{"meta":{"date":"2026-09-30","timezone":"UTC","generated_at":"2026-10-14T13:33:19Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":98,"failed_runs":52,"runs_with_occurrences":52,"runs_with_failure_pattern_matches":48,"runs_with_unmatched_occurrences":4,"runs_without_occurrence_rows":46,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7283/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105443667987664896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7283,"pr_state":"closed","pr_sha":"e7158a2130aaa1ec2b66af7bdf7f99e356001c8e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T23:44:22Z","started_at":"2026-09-30T23:44:22Z","completed_at":"2026-10-01T01:52:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:59:03.291818604Z","timing_metadata_checked_at":"2026-10-01T01:59:03.291818604Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:59:03.295375593Z","region_metadata_checked_at":"2026-10-01T01:59:03.295375593Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a34097ee209422fc4aa983050d49f5e9e7dae90eb79f1f4ddd5f58ce566da701","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7283/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105443667987664896","occurred_at":"2026-09-30T23:44:22Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-01T00:40:12Z\n  Ended:       2026-10-01T00:54:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7664896-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105435642618974208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7126,"pr_state":"closed","pr_sha":"429533a4edff625c1d9b3f963d1f627dd55bfb9e","final_merged_sha":"54589bc135b17d9071a54f907162af4e9fa3ece9","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T23:12:28Z","started_at":"2026-09-30T23:12:28Z","completed_at":"2026-10-01T01:29:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:38:39.982578282Z","timing_metadata_checked_at":"2026-10-01T01:38:39.982578282Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:38:39.985671473Z","region_metadata_checked_at":"2026-10-01T01:38:39.985671473Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105434099047993344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7258,"pr_state":"closed","pr_sha":"d3ebe31be82921b12d03aac77751ad5e1513c442","final_merged_sha":"a68022814b830b9a8334e00ff48929e4987b9c4a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T23:06:20Z","started_at":"2026-09-30T23:06:20Z","completed_at":"2026-10-01T01:34:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:38:40.33068001Z","timing_metadata_checked_at":"2026-10-01T01:38:40.33068001Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:38:40.333467201Z","region_metadata_checked_at":"2026-10-01T01:38:40.333467201Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105433677730156544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7281,"pr_state":"closed","pr_sha":"0f52db4929ddeb744ff754924b061ebef3af2dc2","final_merged_sha":"128ad3152db580e14c5bdced12edb9abbf3f9bab","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T23:04:40Z","started_at":"2026-09-30T23:04:40Z","completed_at":"2026-10-01T01:25:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:28:26.463798129Z","timing_metadata_checked_at":"2026-10-01T01:28:26.463798129Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:28:26.46690662Z","region_metadata_checked_at":"2026-10-01T01:28:26.46690662Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105433504048222208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T23:03:59Z","started_at":"2026-09-30T23:03:59Z","completed_at":"2026-10-01T01:28:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:38:40.750356717Z","timing_metadata_checked_at":"2026-10-01T01:38:40.750356717Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:38:40.752757409Z","region_metadata_checked_at":"2026-10-01T01:38:40.752757409Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105426318702153728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7272,"pr_state":"closed","pr_sha":"0980b4e5b970eb9dc69ab9505f060340b365390e","final_merged_sha":"0980b4e5b970eb9dc69ab9505f060340b365390e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:35:25Z","started_at":"2026-09-30T22:35:25Z","completed_at":"2026-10-01T01:04:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:08:02.801230809Z","timing_metadata_checked_at":"2026-10-01T01:08:02.801230809Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:08:02.804692499Z","region_metadata_checked_at":"2026-10-01T01:08:02.804692499Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"df8c4603df030a88d5df95c36cb06e1893ed209c3161baf36db8944528a92993","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7272/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105426318702153728","occurred_at":"2026-09-30T22:35:25Z","failed_at":"e2e","signature_id":"e5bf27d444d18a4f63bf57e1c97a66cb2cc56175533ecabe1444ad05ef7c4ffa","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc003c2f500\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4t7wr4mxhmz6\" to finish creating: GET https://rp.j2153728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\",\n      \"name\": \"911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:49:39.505681667Z\",\n      \"endTime\": \"2026-10-01T00:08:45.705788813Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4t7wr4mxhmz6--managed/providers/Microsoft.Compute/virtualMachines/u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4t7wr4mxhmz6--managed/providers/Microsoft.Compute/virtualMachines/u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc003c2f500\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4t7wr4mxhmz6\" to finish creating: GET https://rp.j2153728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\",\n      \"name\": \"911cbd71-d1cd-4d26-a932-ae7e6da8fb9f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:49:39.505681667Z\",\n      \"endTime\": \"2026-10-01T00:08:45.705788813Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4t7wr4mxhmz6--managed/providers/Microsoft.Compute/virtualMachines/u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4t7wr4mxhmz6--managed/providers/Microsoft.Compute/virtualMachines/u3l1m0a6z5n1f9b-np-deletable-265vx-bw7dl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4t7wr4mxhmz6\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105425910554431488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7276,"pr_state":"closed","pr_sha":"efea9c8cd78bb4f9a3458bdad0eba29fd92e0ceb","final_merged_sha":"efea9c8cd78bb4f9a3458bdad0eba29fd92e0ceb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:33:48Z","started_at":"2026-09-30T22:33:48Z","completed_at":"2026-10-01T00:50:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:57:52.444286044Z","timing_metadata_checked_at":"2026-10-01T00:57:52.444286044Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:57:52.447721633Z","region_metadata_checked_at":"2026-10-01T00:57:52.447721633Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"342e9dd3aba60c1c3b08bb61deab5f3deb95e74cd918ae4d6fa85d358dc8b32d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7276/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105425910554431488","occurred_at":"2026-09-30T22:33:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105424510625779712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:28:14Z","started_at":"2026-09-30T22:28:14Z","completed_at":"2026-10-01T00:46:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:57:52.157057433Z","timing_metadata_checked_at":"2026-10-01T00:57:52.157057433Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:57:52.160525222Z","region_metadata_checked_at":"2026-10-01T00:57:52.160525222Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c69b45dae2473314b7bc95fb45be89a84b49844b213b2ede53b4bba955bb87b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105424510625779712","occurred_at":"2026-09-30T22:28:14Z","failed_at":"alert","signature_id":"4fb9fed155ed0a95351ac3cfe726d666cbdd3c00e444f36f4da70034c17f36f3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 27 time(s) (20 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-9p7qd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-9p7qd has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-75htw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-75htw has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-hsnlr\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-hsnlr has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-bdq7h\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-bdq7h has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-t7t2f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-t7t2f has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-c472z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-c472z has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-6j8pl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-6j8pl has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-5vmb4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-5vmb4 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-z5vtc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-z5vtc has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-9llk2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-9llk2 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-b79df\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-b79df has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-pgcdv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-pgcdv has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-cm978\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-cm978 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-bb46w\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-bb46w has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-sqbrn\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-sqbrn has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-kmksg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-kmksg has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-s5k4b\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-s5k4b has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-v55n5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-v55n5 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-lxq4s\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-lxq4s has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-30T23:32:33Z\n  Ended:       2026-09-30T23:48:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5779712-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a\", pod=\"ovnkube-control-plane-68b744478b-xfjzk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5kiq11h0qvt8t7eorikel98jokuk8l-p9t6l4h7d3g4h7a/ovnkube-control-plane-68b744478b-xfjzk has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105423700407881728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6580,"pr_state":"open","pr_sha":"1fe9af59ce06c5d6f785a91421edf88db61a778f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T22:25:01Z","started_at":"2026-09-30T22:25:01Z","completed_at":"2026-10-01T01:15:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T01:18:10.855729836Z","timing_metadata_checked_at":"2026-10-01T01:18:10.855729836Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T01:18:10.859444026Z","region_metadata_checked_at":"2026-10-01T01:18:10.859444026Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e0b6183183b94f2be4b8df3aa7e7767697c95cf4ce6304e84c77c9b7c3aa2b17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105423700407881728","occurred_at":"2026-09-30T22:25:01Z","failed_at":"e2e","signature_id":"d39a6f967f0a8906df550b2068f509270953857462cb691e6faf8e691e932263","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:287]: failed to scale down node pool \"np-scale\" from 2 to 1 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00217d260\u003e: \n    failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-d7bh2pbbtldm\" to finish updating, caused by: timeout '60.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-d7bh2pbbtldm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:287]: failed to scale down node pool \"np-scale\" from 2 to 1 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00217d260\u003e: \n    failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-d7bh2pbbtldm\" to finish updating, caused by: timeout '60.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-d7bh2pbbtldm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-d7bh2pbbtldm\" to finish updating, caused by: timeout '60.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-d7bh2pbbtldm, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105423324162035712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"1df99c98be21cd453cc4fb1e16e6b31dea03e43d","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T22:23:31Z","started_at":"2026-09-30T22:23:31Z","completed_at":"2026-10-01T00:41:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:47:04.169055816Z","timing_metadata_checked_at":"2026-10-01T00:47:04.169055816Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:47:04.173073003Z","region_metadata_checked_at":"2026-10-01T00:47:04.173073003Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2cecefa8fdd30629cdccb1c67afa1b35a38aac5f2908474d5bef5942746fc2b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105423324162035712","occurred_at":"2026-09-30T22:23:31Z","failed_at":"alert","signature_id":"d65b8e364e6c08a1a77addfca0b9017dbd9c06c54662364ca5867cb456a5743a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T23:47:21Z\n  Ended:       2026-09-30T23:53:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2035712-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5ki3fusaq8plth807pisqonnpgbdn6-e4t7z1r7k6m9l4f\", pod=\"kube-state-metrics-hcp-754f855b69-774l4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5ki3fusaq8plth807pisqonnpgbdn6-e4t7z1r7k6m9l4f/kube-state-metrics-hcp-754f855b69-774l4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7055/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105421859376861184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7055,"pr_state":"closed","pr_sha":"ea023d8558ae7822bf5aa0216dda1632c30fd364","final_merged_sha":"857119d4c2c287f8a23a5e84260abfca4fbbefe5","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T22:17:42Z","started_at":"2026-09-30T22:17:42Z","completed_at":"2026-10-01T00:34:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:36:09.102899623Z","timing_metadata_checked_at":"2026-10-01T00:36:09.102899623Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:36:09.105885314Z","region_metadata_checked_at":"2026-10-01T00:36:09.105885314Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105417810694901760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"1a558ec553718fc2df21030c08516ca31075af74","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:01:37Z","started_at":"2026-09-30T22:01:37Z","completed_at":"2026-10-01T00:17:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:26:00.091672387Z","timing_metadata_checked_at":"2026-10-01T00:26:00.091672387Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:26:00.094378078Z","region_metadata_checked_at":"2026-10-01T00:26:00.094378078Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"792eec89a9c6b48446a559d6d065521860d8dcee1c0c4bbd602a126af9ad8f63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105417810694901760","occurred_at":"2026-09-30T22:01:37Z","failed_at":"alert","signature_id":"443507f024e81fd0df310f54c24e49a4edd8dadf2d4fcc9aee122ad7ab6fba9d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T23:20:44Z\n  Ended:       2026-09-30T23:26:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4901760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5k6opsebogjptm5uab2qhhik4rqbt7-k4w6n5p5s8z3z6k\", pod=\"azure-file-csi-driver-controller-7c869d77d7-z84jf\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5k6opsebogjptm5uab2qhhik4rqbt7-k4w6n5p5s8z3z6k/azure-file-csi-driver-controller-7c869d77d7-z84jf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7280/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105414632171638784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7280,"pr_state":"closed","pr_sha":"47b7fae8cc2ff19a5b122059ed0b440b2df55933","final_merged_sha":"47b7fae8cc2ff19a5b122059ed0b440b2df55933","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T21:48:59Z","started_at":"2026-09-30T21:48:59Z","completed_at":"2026-10-01T00:06:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:15:58.844101094Z","timing_metadata_checked_at":"2026-10-01T00:15:58.844101094Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:15:58.846697486Z","region_metadata_checked_at":"2026-10-01T00:15:58.846697486Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105413427336843264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7242,"pr_state":"closed","pr_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","final_merged_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T21:44:12Z","started_at":"2026-09-30T21:44:12Z","completed_at":"2026-10-01T00:09:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:15:58.663311453Z","timing_metadata_checked_at":"2026-10-01T00:15:58.663311453Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:15:58.666185045Z","region_metadata_checked_at":"2026-10-01T00:15:58.666185045Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3969ecb7fa93087fd853295b4fb767dd44f90f125b788cb09b6f9788d873de3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105413427336843264","occurred_at":"2026-09-30T21:44:12Z","failed_at":"e2e","signature_id":"fbd3037b7d4daa93d08a18e11bbde460acac79d80087f4b96712fcd02af72604","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7a960\u003e: \n    failed to create HCP cluster ea-listnpdfsm: failed waiting for cluster=\"ea-listnpdfsm\" in resourcegroup=\"ea-list-g57slgdp8bj6\" to finish creating: GET https://rp.j6843264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7a960\u003e: \n    failed to create HCP cluster ea-listnpdfsm: failed waiting for cluster=\"ea-listnpdfsm\" in resourcegroup=\"ea-list-g57slgdp8bj6\" to finish creating: GET https://rp.j6843264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1eed55-2ffc-4dee-9cff-f7085dfb10d8\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}},{"row_id":"c3fc5fe11406b8ed678501f435a56d166537cf9c8f902d062570790192fe0cc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105413427336843264","occurred_at":"2026-09-30T21:44:12Z","failed_at":"e2e","signature_id":"82b07f2201cf39c1115582abd5ee1c7849d4a3ff98ac6b09e90eee9b9379ac77","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cda1a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s9lfl9dtlsjp\" to finish creating: GET https://rp.j6843264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cda1a0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-s9lfl9dtlsjp\" to finish creating: GET https://rp.j6843264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/165fa122-4b97-461c-a51b-3a9f4e951859\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105400327724339200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7279,"pr_state":"closed","pr_sha":"84cd9115e4cbbf3bca7337d7f503e7979ee88733","final_merged_sha":"84cd9115e4cbbf3bca7337d7f503e7979ee88733","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T20:52:09Z","started_at":"2026-09-30T20:52:09Z","completed_at":"2026-09-30T23:10:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T23:12:17.946396898Z","timing_metadata_checked_at":"2026-09-30T23:12:17.946396898Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T23:12:17.949673288Z","region_metadata_checked_at":"2026-09-30T23:12:17.949673288Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"962fba4f31ab6f6767f8cc9b84e9d9cae9a6cb1404ca5bdc7b09e2117437db67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7279/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105400327724339200","occurred_at":"2026-09-30T20:52:09Z","failed_at":"e2e","signature_id":"7b187491ce9c30d3ede87c8dd648a76fcd7a020441133e8aa1cde7d221cdf9af","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-9xv2vk78f5m7\" to finish creating: GET https://rp.j4339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\",\n  \"name\": \"9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-30T22:15:27.259108656Z\",\n  \"endTime\": \"2026-09-30T22:34:33.885780975Z\",\n  \"error\": {\n    \"code\": \"DeadlineExceeded\",\n    \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-9xv2vk78f5m7--managed/providers/Microsoft.Compute/virtualMachines/v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-9xv2vk78f5m7--managed/providers/Microsoft.Compute/virtualMachines/v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79 to exist\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-9xv2vk78f5m7\" to finish creating: GET https://rp.j4339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\",\n  \"name\": \"9f34fa79-f2bb-47f4-8491-50e2c8ff8e21\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-30T22:15:27.259108656Z\",\n  \"endTime\": \"2026-09-30T22:34:33.885780975Z\",\n  \"error\": {\n    \"code\": \"DeadlineExceeded\",\n    \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-9xv2vk78f5m7--managed/providers/Microsoft.Compute/virtualMachines/v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-9xv2vk78f5m7--managed/providers/Microsoft.Compute/virtualMachines/v7o2l4s3b0x1k3m-np-update-nodes-z48kk-nvb79 to exist\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105399910143627264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T20:50:29Z","started_at":"2026-09-30T20:50:29Z","completed_at":"2026-09-30T23:21:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T23:23:07.98556018Z","timing_metadata_checked_at":"2026-09-30T23:23:07.98556018Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T23:23:07.98873487Z","region_metadata_checked_at":"2026-09-30T23:23:07.98873487Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105390838300282880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"7868ef2b26e0bfedc24919742a75cbc3e71f2259","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T20:14:26Z","started_at":"2026-09-30T20:14:26Z","completed_at":"2026-09-30T22:32:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T22:40:15.779814399Z","timing_metadata_checked_at":"2026-09-30T22:40:15.779814399Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T22:40:15.785448682Z","region_metadata_checked_at":"2026-09-30T22:40:15.785448682Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"closed","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"273677ce69aa7a8dc7e1da98b8942d3ea48b24d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T19:51:15Z","started_at":"2026-09-30T19:51:15Z","completed_at":"2026-09-30T20:59:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T21:05:43.346707467Z","timing_metadata_checked_at":"2026-09-30T21:05:43.346707467Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T21:05:43.349835457Z","region_metadata_checked_at":"2026-09-30T21:05:43.349835457Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"0722757c064db6381ff80d4afa66e811aa44bd50b1cd28485390bb091f09770c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","occurred_at":"2026-09-30T19:51:15Z","failed_at":"alert","signature_id":"fa068afb5532b7a0878648a63fa9e7d202e29a9179905dc7ba19af246eee3c2f","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:39:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j8920064-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j8920064-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"85fb91a50462ba78d8fa9e5c07608f6a3bc7db02b136134580ec4b4e48ac5341","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","occurred_at":"2026-09-30T19:51:15Z","failed_at":"alert","signature_id":"4dce540a5737084b6a6f4fb2bff2bdb7b393fc2c015c007f9cfd99a7f9898533","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:39:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8920064/providers/microsoft.monitor/accounts/services-j8920064\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-6rhvw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:40:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8920064/providers/microsoft.monitor/accounts/services-j8920064\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-6rhvw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"995d3be8cce47fb67c1d7f254b2b07dbc1f595f3f31af76e6754f9f78acc3ac3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","occurred_at":"2026-09-30T19:51:15Z","failed_at":"provision","signature_id":"59c91e2e24f0ff8a4982b6cbce1ad6ea4d614e46730790c964f60065e02b662a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T20:23:20.071Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T20:23:20.785Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j8920064-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j8920064-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T20:23:20.785Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:23:21.014Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T20:23:21.176Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T20:23:21.176Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:23:21.232Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:33:24.255Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T20:33:24.255Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T20:34:24.257Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T20:34:24.848Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j8920064-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j8920064-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T20:34:24.848Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:25.188Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T20:34:25.588Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:25.588Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:25.757Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T20:34:26.389Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:26.390Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:26.616Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:26.899Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T20:34:27.340Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T20:34:27.673Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:27.873Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:28.054Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:34:28.402Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T20:34:28.402Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:34:28.487Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T20:44:30.230Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T20:44:30.230Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T20:45:30.230Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T20:45:30.710Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j8920064-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j8920064-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#87a67f37-4acf-40f9-a2aa-33f7bdfa8ae1 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T20:45:30.711Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:31.093Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T20:45:31.467Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:31.467Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:31.591Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T20:45:32.324Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:32.324Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:32.528Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:32.928Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T20:45:33.424Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T20:45:33.656Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:34.121Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:34.518Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T20:45:34.878Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T20:45:34.878Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T20:45:34.950Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T20:53:20.192Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-30T20:53:20.193Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"d9411080f96d720dd47fa0c8fbf8e479c8ada16c452bc602ec4613651fab823b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105385002328920064","occurred_at":"2026-09-30T19:51:15Z","failed_at":"alert","signature_id":"a917e86272dd2f9d0f31aa7b4cfdde08465b4442f6dace77ef110fd12ce6152f","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:29:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-ptxgs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-ptxgs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:29:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-rfxj6\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-rfxj6 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T20:33:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8920064-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-fwc59\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-fwc59 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105384924235173888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"1a558ec553718fc2df21030c08516ca31075af74","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T19:50:56Z","started_at":"2026-09-30T19:50:56Z","completed_at":"2026-09-30T20:39:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:44:46.63787738Z","timing_metadata_checked_at":"2026-09-30T20:44:46.63787738Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:44:46.641242771Z","region_metadata_checked_at":"2026-09-30T20:44:46.641242771Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c09cbf2c3dfce92a53d3d9a1aeed7f2f194fb9514844d7aaa8d393dc3fe2fc49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105384924235173888","occurred_at":"2026-09-30T19:50:56Z","failed_at":"provision","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T20:31:01.398Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-09-30T20:31:01.965Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-09-30T20:31:02.790Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=0a6bcb1a4c8db820be60ebb3431d76a20397d50a3cee281e89a5518a17e2b3e0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5173888-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F0a6bcb1a4c8db820be60ebb3431d76a20397d50a3cee281e89a5518a17e2b3e0\ntime=2026-09-30T20:33:03.396Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5173888-mgmt-2/providers/Microsoft.Resources/deployments/0a6bcb1a4c8db820be60ebb3431d76a20397d50a3cee281e89a5518a17e2b3e0/operationStatuses/08584108066230621918\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5173888-mgmt-2/providers/Microsoft.Resources/deployments/profile-mlyktaqazs6u6\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5173888-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j5173888-mgmt-2/resourceAssociations/nsp-j5173888-mgmt-2-ra-30be3175-d83b-506a-923e-30e4da745648\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"TaskCanceledException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5173888-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j5173888-mgmt-2/resourceAssociations/nsp-j5173888-mgmt-2-ra-30be3175-d83b-506a-923e-30e4da745648, Error message: The request was canceled due to the configured HttpClient.Timeout of 100 seconds elapsing.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f315c2ea7d854194ea48cb15cb5892a7a5211cd6320329993a304b3e4444a395","failure_pattern":"ERROR CODE: DeploymentFailed; detail code TaskCanceledException; detail message Error while processing network security perimeter association: The request was canceled due to the configured HttpClient.Timeout of 100 seconds elapsing.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f315c2ea7d854194ea48cb15cb5892a7a5211cd6320329993a304b3e4444a395"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105382781734359040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"c44277134a494ab263b13a52300b4cbda7bd8fd2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T19:42:25Z","started_at":"2026-09-30T19:42:25Z","completed_at":"2026-09-30T22:03:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T22:19:04.738054667Z","timing_metadata_checked_at":"2026-09-30T22:19:04.738054667Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T22:19:04.742492954Z","region_metadata_checked_at":"2026-09-30T22:19:04.742492954Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e2050a5ba0118e5f25852a3a2793c7f261f53351b316539c9c8b7db25ce200bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105382781734359040","occurred_at":"2026-09-30T19:42:25Z","failed_at":"e2e","signature_id":"44dbc70475d1aef57e4bf508ea5b658bc75ebeac8f3ec4f8affd2ae8a9db5fcb","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:175]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015b4cc0\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: no VMs found for nodepool des-np in managed resource group des-encrypt-gngwklk9wp6n--managed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:175]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015b4cc0\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: no VMs found for nodepool des-np in managed resource group des-encrypt-gngwklk9wp6n--managed\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a90d903a85215a711ab538a5cb9689c2473a833a5956b38bc348d987dab53d7b","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: no VMs found for nodepool \u003cnodepool\u003e in managed resource group \u003cresource-group\u003e","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: no VMs found for nodepool des-np in managed resource group des-encrypt-gngwklk9wp6n--managed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a90d903a85215a711ab538a5cb9689c2473a833a5956b38bc348d987dab53d7b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105370345325203456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"f85307ac47d725462ca085087636a9e9fd03a66e","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T18:52:59Z","started_at":"2026-09-30T18:52:59Z","completed_at":"2026-09-30T21:15:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T21:16:11.864014272Z","timing_metadata_checked_at":"2026-09-30T21:16:11.864014272Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T21:16:11.867277662Z","region_metadata_checked_at":"2026-09-30T21:16:11.867277662Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1b8682400cfc92334cdeaf5257da2e85e5beb45bb2c8ee31331954a074d54b7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105370345325203456","occurred_at":"2026-09-30T18:52:59Z","failed_at":"alert","signature_id":"1e9c1ef55ebe30e66d0a2cb5a25733429fd4759e44b43e5ffc7cc434e6416df0","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T19:59:11Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-x6hb9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-x6hb9 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T19:59:12Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-l2s48\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-l2s48 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T19:59:12Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-4pbr7\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-4pbr7 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T19:59:12Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-wwpwh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-wwpwh has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-30T20:00:11Z\n  Ended:       2026-09-30T20:14:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5203456-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p\", pod=\"control-plane-operator-d4ff858b6-tvjkl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5hgejihc22takh2irlnl38m4fd005m-q6p1l2n2a3b6z5p/control-plane-operator-d4ff858b6-tvjkl has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105360937161592832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"closed","pr_sha":"425c4c356c64b8430e722d04ae7e02d916e5788f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T18:15:37Z","started_at":"2026-09-30T18:15:37Z","completed_at":"2026-09-30T20:32:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:44:46.738717205Z","timing_metadata_checked_at":"2026-09-30T20:44:46.738717205Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:44:46.741811696Z","region_metadata_checked_at":"2026-09-30T20:44:46.741811696Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105360377792434176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"c1711e8366a51583591e5ff0550d5c37cc22c519","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T18:13:24Z","started_at":"2026-09-30T18:13:24Z","completed_at":"2026-09-30T20:32:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:44:46.832290649Z","timing_metadata_checked_at":"2026-09-30T20:44:46.832290649Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:44:46.834941942Z","region_metadata_checked_at":"2026-09-30T20:44:46.834941942Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e7967285c75d89eb2d1af2ea10f05b262bb0f8af6441b0030103b65f454d4b32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105360377792434176","occurred_at":"2026-09-30T18:13:24Z","failed_at":"e2e","signature_id":"ef889d6333cbd7f0e4691f6a9291bdf5c9b96b5e6e9bbbf10e509094968d5d90","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001666348\u003e: \n    resource group \"rg-candidate-5-1-7k2hspfgwlgc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-jdl45b\" in resourcegroup=\"rg-candidate-5-1-7k2hspfgwlgc\" to finish deleting: GET https://rp.j2434176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77026e76-9732-4942-8d30-76c6571b6048\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77026e76-9732-4942-8d30-76c6571b6048\",\n      \"name\": \"77026e76-9732-4942-8d30-76c6571b6048\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T19:19:44.434042017Z\",\n      \"endTime\": \"2026-09-30T19:43:54.082755547Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5gr9ar4ha50b9n6jmkh1qfntpjdb09 still exists (deletion dispatched at 2026-09-30T19:19:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001666348\u003e: \n    resource group \"rg-candidate-5-1-7k2hspfgwlgc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-jdl45b\" in resourcegroup=\"rg-candidate-5-1-7k2hspfgwlgc\" to finish deleting: GET https://rp.j2434176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77026e76-9732-4942-8d30-76c6571b6048\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77026e76-9732-4942-8d30-76c6571b6048\",\n      \"name\": \"77026e76-9732-4942-8d30-76c6571b6048\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T19:19:44.434042017Z\",\n      \"endTime\": \"2026-09-30T19:43:54.082755547Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5gr9ar4ha50b9n6jmkh1qfntpjdb09 still exists (deletion dispatched at 2026-09-30T19:19:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105355115949985792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"8470e4a8152fec1ad8e98ecdb32250dd598096a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T17:52:29Z","started_at":"2026-09-30T17:52:29Z","completed_at":"2026-09-30T20:12:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:13:49.414194768Z","timing_metadata_checked_at":"2026-09-30T20:13:49.414194768Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:13:49.418102556Z","region_metadata_checked_at":"2026-09-30T20:13:49.418102556Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"closed","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"273677ce69aa7a8dc7e1da98b8942d3ea48b24d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T17:50:19Z","started_at":"2026-09-30T17:50:19Z","completed_at":"2026-09-30T19:01:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:12:03.806145761Z","timing_metadata_checked_at":"2026-09-30T19:12:03.806145761Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:12:03.809184752Z","region_metadata_checked_at":"2026-09-30T19:12:03.809184752Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"14158fca6a762126957986a1d16387e11ebf35bb70d83ddc29974a92e17ab902","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","occurred_at":"2026-09-30T17:50:19Z","failed_at":"provision","signature_id":"7c9fdc5e068f8cbc4ff91d33bd3a3416cfd241a4e1d4d0d28ad54d1ce1f099cd","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T18:26:03.649Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T18:26:04.095Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0633472-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j0633472-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:42f17d2a-2d54-4fa2-b94e-06bedcfbde0d region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#42f17d2a-2d54-4fa2-b94e-06bedcfbde0d INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T18:26:04.095Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:26:04.389Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T18:26:04.602Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T18:26:04.602Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:26:04.633Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:36:07.312Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T18:36:07.312Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T18:37:07.314Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T18:37:08.026Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0633472-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j0633472-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:42f17d2a-2d54-4fa2-b94e-06bedcfbde0d region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#42f17d2a-2d54-4fa2-b94e-06bedcfbde0d INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T18:37:08.026Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:08.359Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T18:37:08.708Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:08.708Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:08.746Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T18:37:09.698Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:09.698Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:09.874Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:10.131Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T18:37:10.610Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T18:37:10.790Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:11.050Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:11.400Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:37:11.717Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T18:37:11.717Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:37:11.766Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T18:47:13.248Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T18:47:13.248Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T18:48:13.252Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T18:48:13.967Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j0633472-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j0633472-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:42f17d2a-2d54-4fa2-b94e-06bedcfbde0d region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#42f17d2a-2d54-4fa2-b94e-06bedcfbde0d INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T18:48:13.968Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:14.374Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T18:48:14.654Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:14.654Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:14.778Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T18:48:15.694Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:15.694Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:16.096Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:16.376Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T18:48:16.694Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T18:48:16.931Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:17.265Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:17.464Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T18:48:17.811Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T18:48:17.811Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T18:48:17.876Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T18:56:03.654Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-30T18:56:03.655Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"28bef2f8716812d67b6e6e25ea19c6bc93f6c2c5a7ff92017c953f4ceea28349","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","occurred_at":"2026-09-30T17:50:19Z","failed_at":"alert","signature_id":"73766b39b2ffbd664b524f7f700b76efc77e57dbad9d236a23c2d8e65fc0b9c5","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:42:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j0633472-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j0633472-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"29c5a30ffae7d6e99665345e5addd700d58862d7e20b9c55632ac25b8a33f1f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","occurred_at":"2026-09-30T17:50:19Z","failed_at":"alert","signature_id":"c7530f8f951fb0bc0dedf42f576c3e580237189727a6abff781b0abf7fea08d7","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:42:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.92:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0633472/providers/microsoft.monitor/accounts/services-j0633472\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-kq5jf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:42:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.92:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0633472/providers/microsoft.monitor/accounts/services-j0633472\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-kq5jf\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"ec3893c806905c5531245d0f54e9fcc5d4e2aa61eedd85b78c44b2478b367181","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354569830633472","occurred_at":"2026-09-30T17:50:19Z","failed_at":"alert","signature_id":"032b9ffd2c84a0fa436845d9647a87344f7d1ada690aa0f9f17c34f86c623a85","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-qp5bd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-qp5bd has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-x78jx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-x78jx has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:32:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0633472-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-55p2d\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-55p2d has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354557851701248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"e01d7004aa224ff873a1a813ac3e6932792e1135","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T17:50:16Z","started_at":"2026-09-30T17:50:16Z","completed_at":"2026-09-30T20:15:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:24:17.750435178Z","timing_metadata_checked_at":"2026-09-30T20:24:17.750435178Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:24:17.75289927Z","region_metadata_checked_at":"2026-09-30T20:24:17.75289927Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c77b3cc9745bdc781d76c5355ec6a8ce5bf9242bcd16b866a1c61d364f5088cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354557851701248","occurred_at":"2026-09-30T17:50:16Z","failed_at":"e2e","signature_id":"a106d5350ee15080f79123cc2e755f305356aa43155c5554eb5bbed8c6fbadba","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014f6680\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mxcczjbklf6k\" to finish creating: GET https://rp.j1701248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49794eca-872f-4949-8230-a9c6d512647a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49794eca-872f-4949-8230-a9c6d512647a\",\n      \"name\": \"49794eca-872f-4949-8230-a9c6d512647a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T19:06:11.188459377Z\",\n      \"endTime\": \"2026-09-30T19:25:16.297661458Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine p5d6o0j0i5s4u7a-autoscale-noaz-686hx-dvr6p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine p5d6o0j0i5s4u7a-autoscale-noaz-686hx-dvr6p: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014f6680\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mxcczjbklf6k\" to finish creating: GET https://rp.j1701248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49794eca-872f-4949-8230-a9c6d512647a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49794eca-872f-4949-8230-a9c6d512647a\",\n      \"name\": \"49794eca-872f-4949-8230-a9c6d512647a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T19:06:11.188459377Z\",\n      \"endTime\": \"2026-09-30T19:25:16.297661458Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine p5d6o0j0i5s4u7a-autoscale-noaz-686hx-dvr6p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine p5d6o0j0i5s4u7a-autoscale-noaz-686hx-dvr6p: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T17:50:11Z","started_at":"2026-09-30T17:50:11Z","completed_at":"2026-09-30T20:20:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:24:17.473007942Z","timing_metadata_checked_at":"2026-09-30T20:24:17.473007942Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:24:17.476206332Z","region_metadata_checked_at":"2026-09-30T20:24:17.476206332Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1c6e9e913497fc9db7a8b9f4487cd7d0d87ab4e1c2dc1aaa6c7ed856afe9b538","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"e2e","signature_id":"32302e8488c217b3d6d677ca11a50f92f8cc2fff814de0e9b445cdfbf18e3592","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e5b548\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e5b548\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-46kd7.apps.aro.cilium-cl.f3k6.j5347072.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"566ab055016e04a6701fbec38885336198599e54140b613d9b347e299b8a5fee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"alert","signature_id":"1b88ffe98af3a32b79edf3e358fff5366d78c3eb3856b592d15721147e9bc14e","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T19:14:32Z\n  Ended:       2026-09-30T19:40:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"009dd713-cc58-4cfb-b82f-f1fca7b413d6\"\n  Description: Pod ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T19:14:32Z\n  Ended:       2026-09-30T19:41:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"009dd713-cc58-4cfb-b82f-f1fca7b413d6\"\n  Description: Pod ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"6135af3e05ea7f621c994c5205e2c81597fd1be72ef583fa17eb41bdf2f67830","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"e2e","signature_id":"f4f59b5ba44666f24d23e97f157df80ef1f26ced02f1c0c9bfaab2ad9f5e855b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001128760\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-h59xkph9dfpm\" to finish creating: GET https://rp.j5347072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88da31a3-069e-4240-80a2-676685212f2c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88da31a3-069e-4240-80a2-676685212f2c\",\n      \"name\": \"88da31a3-069e-4240-80a2-676685212f2c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T18:49:41.355729108Z\",\n      \"endTime\": \"2026-09-30T19:18:42.930044888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001128760\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-h59xkph9dfpm\" to finish creating: GET https://rp.j5347072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88da31a3-069e-4240-80a2-676685212f2c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88da31a3-069e-4240-80a2-676685212f2c\",\n      \"name\": \"88da31a3-069e-4240-80a2-676685212f2c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T18:49:41.355729108Z\",\n      \"endTime\": \"2026-09-30T19:18:42.930044888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8404bee177ab7afaec301e103109170a41a02e3a81b48d821068765798f1085d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"alert","signature_id":"dabbf455e614a0564c0abb3cb0bd1b76792259597e5b2a81a72dd954978bbab0","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T19:11:34Z\n  Ended:       2026-09-30T19:36:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72jx2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T19:22:36Z\n  Ended:       2026-09-30T19:36:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72jx2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"e42cf3abc9f03f5b7b9e323f1771245d9d08937d0afa4e25f7310c551f77f01e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354536905347072","occurred_at":"2026-09-30T17:50:11Z","failed_at":"alert","signature_id":"36bab4fa8a374fa0764d196f80453ffc18fe93b52d905fcc5c5da63a9d7c7d96","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T19:25:32Z\n  Ended:       2026-09-30T19:38:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72jx2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T19:25:32Z\n  Ended:       2026-09-30T19:38:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5347072-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.33:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5347072/providers/microsoft.monitor/accounts/hcps-j5347072\", namespace=\"ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72jx2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5gnhij1pbtee5r1icg9qlas1lv2h5j-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354517317947392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"7551c0170c5cffea9e64f3707dcf2378a40b7d3f","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T17:50:07Z","started_at":"2026-09-30T17:50:07Z","completed_at":"2026-09-30T18:10:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:29:07.480883426Z","timing_metadata_checked_at":"2026-09-30T18:29:07.480883426Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-30T18:29:07.486962108Z","region_metadata_checked_at":"2026-09-30T19:32:30.846466611Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fa7a70ab2f51e58bce1f4a1e8bee6dab536edb6e683c9d8af5536f7a1bc47862","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354517317947392","occurred_at":"2026-09-30T17:50:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105354426800672768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T17:49:45Z","started_at":"2026-09-30T17:49:45Z","completed_at":"2026-09-30T20:23:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:24:17.636537032Z","timing_metadata_checked_at":"2026-09-30T20:24:17.636537032Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:24:17.639296924Z","region_metadata_checked_at":"2026-09-30T20:24:17.639296924Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6580,"pr_state":"open","pr_sha":"4aa6a298e0b5fbbd9a3319bd9a8e7be421dc6b6b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T17:30:58Z","started_at":"2026-09-30T17:30:58Z","completed_at":"2026-09-30T19:49:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:53:12.389127734Z","timing_metadata_checked_at":"2026-09-30T19:53:12.389127734Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:53:12.391933027Z","region_metadata_checked_at":"2026-09-30T19:53:12.391933027Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1bfd92995ffc627a5bcbe66840c6bf30efbf0a71b9da37c7bce568acafa17f69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"alert","signature_id":"2850f1981822705d765169c8df6b099246627ba71fa905aafb70cde8db42aecd","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T18:52:59Z\n  Ended:       2026-09-30T19:08:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72qr2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T18:52:59Z\n  Ended:       2026-09-30T19:08:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72qr2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"267cee4456e6c477ccdbe87f52940b12a2799e6a4afe3ae2140c814e0662f13d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"alert","signature_id":"27312778a2840cef9f5cf9f4455e8cbaa9b89abd17091183ef5636ac2a6c1aae","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T18:43:56Z\n  Ended:       2026-09-30T19:12:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bce6bb08-1647-4613-853e-b0a5759423ff\"\n  Description: Pod ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T18:44:57Z\n  Ended:       2026-09-30T19:12:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"bce6bb08-1647-4613-853e-b0a5759423ff\"\n  Description: Pod ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"356bcc259121de6bd3c43a3674190686c2ba703112074cf43c432506d492b083","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"e2e","signature_id":"67c7217815599c64efc725333fb06153f8b3a9a0dd9d914a2a506c84453c1f10","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00120c640\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mnp9svrwqcpp\" to finish creating: GET https://rp.j4426752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\",\n      \"name\": \"9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T18:21:17.653151468Z\",\n      \"endTime\": \"2026-09-30T18:50:22.965863416Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00120c640\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mnp9svrwqcpp\" to finish creating: GET https://rp.j4426752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\",\n      \"name\": \"9fd78a0f-65d2-4ffb-8dde-60a82af78dd0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T18:21:17.653151468Z\",\n      \"endTime\": \"2026-09-30T18:50:22.965863416Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5bf47199bedc79b5cd3953894021668017d49701cfec73096b581d00fb1e2f6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"alert","signature_id":"d640d21aa16253cbebe72dcc2119bb10a7a01ae60ba5700101a61e6f4ab54852","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T18:57:01Z\n  Ended:       2026-09-30T19:10:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72qr2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T18:57:01Z\n  Ended:       2026-09-30T19:10:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4426752-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4426752/providers/microsoft.monitor/accounts/hcps-j4426752\", namespace=\"ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-72qr2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5ga8d53uj3o7nh34nh77qbpeiclu0r-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"f0b04248e5a2d8efadd4fe166c99c547bc35e59d6117b6fa535e391c14056985","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105349698964426752","occurred_at":"2026-09-30T17:30:58Z","failed_at":"e2e","signature_id":"5516c4e0d66f0bcaac8052c3730dabeb8f387df4e3055ca5a2bb6566c65eeb38","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:226]: Timed out after 300.001s.\nKAS→kubelet log path never became reachable from VM inside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:220 with:\nno running pods found in openshift-kube-apiserver\nExpected\n    \u003cstring\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:226]: Timed out after 300.001s.\nKAS→kubelet log path never became reachable from VM inside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:220 with:\nno running pods found in openshift-kube-apiserver\nExpected\n    \u003cstring\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d4e0f7e2fc9458905eb66d446ace1531b88fae886b89c70332091e7cd2070cd","failure_pattern":"no running pods found in openshift-kube-apiserver","search_query":"no running pods found in openshift-kube-apiserver"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0d4e0f7e2fc9458905eb66d446ace1531b88fae886b89c70332091e7cd2070cd","13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105348343122104320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7258,"pr_state":"closed","pr_sha":"8f1369d67221dfa00a0bb0152a41464a28923688","final_merged_sha":"a68022814b830b9a8334e00ff48929e4987b9c4a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T17:25:35Z","started_at":"2026-09-30T17:25:35Z","completed_at":"2026-09-30T19:53:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T20:13:49.840620519Z","timing_metadata_checked_at":"2026-09-30T20:13:49.840620519Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T20:13:49.843437211Z","region_metadata_checked_at":"2026-09-30T20:13:49.843437211Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105342498506280960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"1a558ec553718fc2df21030c08516ca31075af74","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T17:02:21Z","started_at":"2026-09-30T17:02:21Z","completed_at":"2026-09-30T19:14:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:22:23.075444721Z","timing_metadata_checked_at":"2026-09-30T19:22:23.075444721Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:22:23.078359112Z","region_metadata_checked_at":"2026-09-30T19:22:23.078359112Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7b1561121e2386a6ad753da0ba9bc1fd5adf3f6e94fb4bba0b75cbd7d93a0da0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105342498506280960","occurred_at":"2026-09-30T17:02:21Z","failed_at":"alert","signature_id":"43d2c6ba277aa8bbbc5cc8bc7b5f2aa98baae126a8f1b3317711f9e7e5ae87d9","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:57:46Z\n  Ended:       2026-09-30T18:21:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6280960-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5fn1t8q899f67e9c3gfapqs0nka9q4-z9j3x8s3e4f8g8i\", pod=\"azure-file-csi-driver-operator-6fc68567db-xb6kc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5fn1t8q899f67e9c3gfapqs0nka9q4-z9j3x8s3e4f8g8i/azure-file-csi-driver-operator-6fc68567db-xb6kc has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:57:46Z\n  Ended:       2026-09-30T18:21:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6280960-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5fn1t8q899f67e9c3gfapqs0nka9q4-z9j3x8s3e4f8g8i\", pod=\"azure-file-csi-driver-operator-6fc68567db-ck5hv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5fn1t8q899f67e9c3gfapqs0nka9q4-z9j3x8s3e4f8g8i/azure-file-csi-driver-operator-6fc68567db-ck5hv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105340384333795328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7271,"pr_state":"open","pr_sha":"0e6699ba542781a024685f4459df8366ac7e7758","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T16:53:57Z","started_at":"2026-09-30T16:53:57Z","completed_at":"2026-09-30T19:09:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:12:04.050578011Z","timing_metadata_checked_at":"2026-09-30T19:12:04.050578011Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:12:04.053520602Z","region_metadata_checked_at":"2026-09-30T19:12:04.053520602Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"76c8cf1608edddbc29c933ced50fbec3c406eea6fad1853a341c19c22b4c9c58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7271/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105340384333795328","occurred_at":"2026-09-30T16:53:57Z","failed_at":"alert","signature_id":"c43369e4c310c6727d22ed8986f0196b36cfda0efe919bb5036a8c8d32b2a996","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:53:32Z\n  Ended:       2026-09-30T18:09:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3795328-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:56:30Z\n  Ended:       2026-09-30T18:09:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3795328-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T17:59:28Z\n  Ended:       2026-09-30T18:09:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3795328-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105335648247353344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6807,"pr_state":"open","pr_sha":"e0f7c1daf2fdc08782c031a5b04a7bc7fcfc0c90","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T16:35:08Z","started_at":"2026-09-30T16:35:08Z","completed_at":"2026-09-30T18:56:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:01:24.217237684Z","timing_metadata_checked_at":"2026-09-30T19:01:24.217237684Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:01:24.220514676Z","region_metadata_checked_at":"2026-09-30T19:01:24.220514676Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105334781733507072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"closed","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"d98ef6349a3cd6d57591590dbb9cbff37ac89433","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T16:31:41Z","started_at":"2026-09-30T16:31:41Z","completed_at":"2026-09-30T18:47:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:50:38.747004045Z","timing_metadata_checked_at":"2026-09-30T18:50:38.747004045Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:50:38.750062536Z","region_metadata_checked_at":"2026-09-30T18:50:38.750062536Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105331328655298560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7124,"pr_state":"closed","pr_sha":"60e1aa40eaea69e03059bb27ebcf9932e637a29a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T16:17:58Z","started_at":"2026-09-30T16:17:58Z","completed_at":"2026-09-30T18:27:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:39:57.236875773Z","timing_metadata_checked_at":"2026-09-30T18:39:57.236875773Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:39:57.24109076Z","region_metadata_checked_at":"2026-09-30T18:39:57.24109076Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"18be967efb11aaa4a6d65c45053c32e6308b3dd467a4a5ce9fe5869c24c11e86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105331328655298560","occurred_at":"2026-09-30T16:17:58Z","failed_at":"alert","signature_id":"a0c6a2c52a8e1e3c5b6144fde866a89d2692c5d407e38e01f2154424838e188d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:13:21Z\n  Ended:       2026-09-30T17:39:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5298560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f514d5mgc9haie801lsop9ne436rb-q0n1v0q8g3o5z5q\", pod=\"router-8445c79445-9rnr2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f514d5mgc9haie801lsop9ne436rb-q0n1v0q8g3o5z5q/router-8445c79445-9rnr2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"457f30ea1c767424543b62f81eb28c323addd88bbde44ce0671f5ee57f5c75cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105331328655298560","occurred_at":"2026-09-30T16:17:58Z","failed_at":"e2e","signature_id":"65e42ab979836c2386f9206947aca6b2f3f248b30240970282f08d100e07f667","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-z8sw852rvcl6/cluster-candidate-5-1-f8bld9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f24640\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-f8bld9: failed waiting for cluster=\"cluster-candidate-5-1-f8bld9\" in resourcegroup=\"rg-candidate-5-1-z8sw852rvcl6\" to finish creating: GET https://rp.j5298560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1da1b003-1c66-47c8-99b2-f49235ea2736\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1da1b003-1c66-47c8-99b2-f49235ea2736\",\n      \"name\": \"1da1b003-1c66-47c8-99b2-f49235ea2736\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:01:56.141378012Z\",\n      \"endTime\": \"2026-09-30T17:30:57.762149233Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 24m10s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-z8sw852rvcl6/cluster-candidate-5-1-f8bld9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f24640\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-f8bld9: failed waiting for cluster=\"cluster-candidate-5-1-f8bld9\" in resourcegroup=\"rg-candidate-5-1-z8sw852rvcl6\" to finish creating: GET https://rp.j5298560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1da1b003-1c66-47c8-99b2-f49235ea2736\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1da1b003-1c66-47c8-99b2-f49235ea2736\",\n      \"name\": \"1da1b003-1c66-47c8-99b2-f49235ea2736\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:01:56.141378012Z\",\n      \"endTime\": \"2026-09-30T17:30:57.762149233Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 24m10s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105330420139692032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T16:14:20Z","started_at":"2026-09-30T16:14:20Z","completed_at":"2026-09-30T18:42:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:50:38.978983059Z","timing_metadata_checked_at":"2026-09-30T18:50:38.978983059Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:50:38.981589351Z","region_metadata_checked_at":"2026-09-30T18:50:38.981589351Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"6f8eb9e8dcaa57d6bdd65df600b63d5e8d60c3a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T16:03:39Z","started_at":"2026-09-30T16:03:39Z","completed_at":"2026-09-30T18:34:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:50:39.153609543Z","timing_metadata_checked_at":"2026-09-30T18:50:39.153609543Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:50:39.156400535Z","region_metadata_checked_at":"2026-09-30T18:50:39.156400535Z"},"failed_at":["alert","e2e"],"failed_test_count":23,"occurrences":[{"row_id":"01cd1bd1a0e51da6c975e3de44ef08f2ccaaa72333ccf4360021bb2b20b01d43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"5fddfe712c98fb0e643ec5bb4c03b94974770b44c356b20f027ad56567a251f3","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:15:29Z\n  Ended:       2026-09-30T17:26:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"prometheus\", node=\"aks-userswft1-23433410-vmss000001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-23433410-vmss000001 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:16:26Z\n  Ended:       2026-09-30T17:26:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"prometheus\", node=\"aks-userswft1-23433410-vmss000001\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-23433410-vmss000001 is unreachable and some workloads may be rescheduled.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired"}},{"row_id":"04f5f2b3781044e3f493d68ba681e321d09abc88943822ede4f3964816b07718","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"e1a6da9fb27aa2281cf1d566ff4e3821f5a89e8834f371968aac14a196fb4167","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104c300\u003e: \n    resource group \"nodepool-delete-c4hb66lp5frs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c4hb66lp5frs\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e6cc5ae-d28a-4989-ad49-e0dd769288e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e6cc5ae-d28a-4989-ad49-e0dd769288e5\",\n      \"name\": \"7e6cc5ae-d28a-4989-ad49-e0dd769288e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:25:05.3374218Z\",\n      \"endTime\": \"2026-09-30T17:49:05.822251371Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f18hpo95treletippqdt0q6nd6217 still exists (deletion dispatched at 2026-09-30T17:25:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104c300\u003e: \n    resource group \"nodepool-delete-c4hb66lp5frs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-c4hb66lp5frs\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e6cc5ae-d28a-4989-ad49-e0dd769288e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e6cc5ae-d28a-4989-ad49-e0dd769288e5\",\n      \"name\": \"7e6cc5ae-d28a-4989-ad49-e0dd769288e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:25:05.3374218Z\",\n      \"endTime\": \"2026-09-30T17:49:05.822251371Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f18hpo95treletippqdt0q6nd6217 still exists (deletion dispatched at 2026-09-30T17:25:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b8bc93626235d4cb9fc042f927def82b4b8c61f1df6f1d07b269cb961f65adc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"0e1182513eb29f65f3a7935b97b78d6d7dd25e1ef38c392477b0b0539d97cd71","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:40:28Z\n  Ended:       2026-09-30T18:18:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0hjlm2j48ooq92uj0su5rbnsq03t-x3y7i6e5z1z9c0c\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t5f0hjlm2j48ooq92uj0su5rbnsq03t-x3y7i6e5z1z9c0c (management cluster ci01-j5446016-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}},{"row_id":"161e5eefe0ad127bda4f2e3d03f6fb4b357e2d45fce6bbb4f54017a825c72366","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"7eadcca87592abc851861d4b05dac9a3fd2f7a948247bc541b68859484eb6b48","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171ee58\u003e: \n    resource group \"v20261001preview-8xzpnp7tdgz2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-8xzpnp7tdgz2\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b4e803e-d3df-4fa2-8617-7c767d9fdb48\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b4e803e-d3df-4fa2-8617-7c767d9fdb48\",\n      \"name\": \"7b4e803e-d3df-4fa2-8617-7c767d9fdb48\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:45.793371573Z\",\n      \"endTime\": \"2026-09-30T17:44:45.916509607Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0lc5egjrilphmb608mchbeokkhli still exists (deletion dispatched at 2026-09-30T17:20:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00171ee58\u003e: \n    resource group \"v20261001preview-8xzpnp7tdgz2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-8xzpnp7tdgz2\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b4e803e-d3df-4fa2-8617-7c767d9fdb48\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b4e803e-d3df-4fa2-8617-7c767d9fdb48\",\n      \"name\": \"7b4e803e-d3df-4fa2-8617-7c767d9fdb48\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:45.793371573Z\",\n      \"endTime\": \"2026-09-30T17:44:45.916509607Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0lc5egjrilphmb608mchbeokkhli still exists (deletion dispatched at 2026-09-30T17:20:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"176744d5c830be212abb122f4d0ed0fd6dbf2195aae13f216403ed952b53ca6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"0a60267a540ef98cc9481c666f895630a98788d08afe3a5800859b181c2717b2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af62c0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fs49jjvbxrlk\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9c1f57e-28e9-49ea-8226-45b731c2d8ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9c1f57e-28e9-49ea-8226-45b731c2d8ca\",\n      \"name\": \"e9c1f57e-28e9-49ea-8226-45b731c2d8ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:53:07.232391508Z\",\n      \"endTime\": \"2026-09-30T17:22:15.941466136Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m42s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000af62c0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-fs49jjvbxrlk\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9c1f57e-28e9-49ea-8226-45b731c2d8ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9c1f57e-28e9-49ea-8226-45b731c2d8ca\",\n      \"name\": \"e9c1f57e-28e9-49ea-8226-45b731c2d8ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:53:07.232391508Z\",\n      \"endTime\": \"2026-09-30T17:22:15.941466136Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m42s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1df8570fe4ddd30d7950398a45af3c00aeeef658cb5f618edc4bb275b9484eb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"94689deb46415801292ac0e274c7ee723941c2649c96c6c36d110ba269a8a320","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:16:20Z\n  Ended:       2026-09-30T17:27:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:16:20Z\n  Ended:       2026-09-30T17:27:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T17:16:20Z\n  Ended:       2026-09-30T17:26:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T17:16:20Z\n  Ended:       2026-09-30T17:26:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired"}},{"row_id":"22c603dee53dcfad2f214c15bb01b8a02db7128ca89c0e8a43ca2e19b9390f7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"9819845289ed857fc47e38f741826a409f9a62a82fd091add158c84457fae98f","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002518090\u003e: \n    resource group \"node-limit-test-2jbt488tkpl7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2jbt488tkpl7\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da56d4ee-d492-4bc3-b379-32ba10c1db68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da56d4ee-d492-4bc3-b379-32ba10c1db68\",\n      \"name\": \"da56d4ee-d492-4bc3-b379-32ba10c1db68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:02:22.062008253Z\",\n      \"endTime\": \"2026-09-30T17:26:25.935485649Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1mqu517if66tcq7vv1883rm6svr2 still exists (deletion dispatched at 2026-09-30T17:02:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002518090\u003e: \n    resource group \"node-limit-test-2jbt488tkpl7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2jbt488tkpl7\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da56d4ee-d492-4bc3-b379-32ba10c1db68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/da56d4ee-d492-4bc3-b379-32ba10c1db68\",\n      \"name\": \"da56d4ee-d492-4bc3-b379-32ba10c1db68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:02:22.062008253Z\",\n      \"endTime\": \"2026-09-30T17:26:25.935485649Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1mqu517if66tcq7vv1883rm6svr2 still exists (deletion dispatched at 2026-09-30T17:02:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2636bbb6c17309d38e7b515826e59b6a3710754313e023bd18317ae3d5124860","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"80e82fe0ee940a917440e65af57e56d56e909ddf306b97e303c0192cedaf42f9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b32210\u003e: \n    resource group \"rg-candidate-4-21-fjd2gblvhgpk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-jb8n5l\" in resourcegroup=\"rg-candidate-4-21-fjd2gblvhgpk\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cbfc4ac-c696-45d2-b553-5a272022650d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cbfc4ac-c696-45d2-b553-5a272022650d\",\n      \"name\": \"9cbfc4ac-c696-45d2-b553-5a272022650d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:14:14.963393723Z\",\n      \"endTime\": \"2026-09-30T17:38:16.020897196Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0e8dbaqbo8t5dt5vdcasi8k08a4q still exists (deletion dispatched at 2026-09-30T17:14:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b32210\u003e: \n    resource group \"rg-candidate-4-21-fjd2gblvhgpk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-jb8n5l\" in resourcegroup=\"rg-candidate-4-21-fjd2gblvhgpk\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cbfc4ac-c696-45d2-b553-5a272022650d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cbfc4ac-c696-45d2-b553-5a272022650d\",\n      \"name\": \"9cbfc4ac-c696-45d2-b553-5a272022650d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:14:14.963393723Z\",\n      \"endTime\": \"2026-09-30T17:38:16.020897196Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0e8dbaqbo8t5dt5vdcasi8k08a4q still exists (deletion dispatched at 2026-09-30T17:14:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2a3da94daddf8b1ccb6eb62aaba296e48b5425986f4a28d44a24931db1d2e345","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"9980c378ec40833f79eaebfd6e66376e813784c1f455cf437569399d4dfcefb8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00280c2a0\u003e: \n    resource group \"rg-candidate-5-0-ns2cs4dsdbxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-spnhk7\" in resourcegroup=\"rg-candidate-5-0-ns2cs4dsdbxw\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7972329-b9a1-4d78-b6ee-4512a34c9cf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7972329-b9a1-4d78-b6ee-4512a34c9cf8\",\n      \"name\": \"c7972329-b9a1-4d78-b6ee-4512a34c9cf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:04.153634774Z\",\n      \"endTime\": \"2026-09-30T17:44:05.883180823Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0ebuj103h4dn4jek0oofbev66a3k still exists (deletion dispatched at 2026-09-30T17:20:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00280c2a0\u003e: \n    resource group \"rg-candidate-5-0-ns2cs4dsdbxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-spnhk7\" in resourcegroup=\"rg-candidate-5-0-ns2cs4dsdbxw\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7972329-b9a1-4d78-b6ee-4512a34c9cf8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7972329-b9a1-4d78-b6ee-4512a34c9cf8\",\n      \"name\": \"c7972329-b9a1-4d78-b6ee-4512a34c9cf8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:04.153634774Z\",\n      \"endTime\": \"2026-09-30T17:44:05.883180823Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0ebuj103h4dn4jek0oofbev66a3k still exists (deletion dispatched at 2026-09-30T17:20:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2e40c647adb2db4e6382852fc257ee6c9cadbae63e46de502a2b332806252618","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"b8d7fd2de97ced8753171c37ab1fa36055ef2afb986b3a9217669c39fc61ff3d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-9z27ks86nc27/cluster-candidate-5-1-hrwtgg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a04c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-hrwtgg: failed waiting for cluster=\"cluster-candidate-5-1-hrwtgg\" in resourcegroup=\"rg-candidate-5-1-9z27ks86nc27\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9135939f-e992-427f-be1e-4ecfae2b319a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9135939f-e992-427f-be1e-4ecfae2b319a\",\n      \"name\": \"9135939f-e992-427f-be1e-4ecfae2b319a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:29.681806994Z\",\n      \"endTime\": \"2026-09-30T17:21:35.77427567Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 22m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-9z27ks86nc27/cluster-candidate-5-1-hrwtgg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a04c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-hrwtgg: failed waiting for cluster=\"cluster-candidate-5-1-hrwtgg\" in resourcegroup=\"rg-candidate-5-1-9z27ks86nc27\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9135939f-e992-427f-be1e-4ecfae2b319a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9135939f-e992-427f-be1e-4ecfae2b319a\",\n      \"name\": \"9135939f-e992-427f-be1e-4ecfae2b319a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:29.681806994Z\",\n      \"endTime\": \"2026-09-30T17:21:35.77427567Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-ec.1 is Partial (want Completed), started 22m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4e3ab6b49e6567cc40cafa154346d25ef5bfd7818598387865a31b0275872eb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"0d33f411900228bb6fdc790337157ac99d17d0d26ebbe8f2667d92df7f4ae88f","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:22:21Z\n  Ended:       2026-09-30T17:28:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:22:21Z\n  Ended:       2026-09-30T17:28:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T17:22:21Z\n  Ended:       2026-09-30T17:28:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T17:22:21Z\n  Ended:       2026-09-30T17:28:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"set-kubelet-parameters-for-scale\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/services-j5446016\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/set-kubelet-parameters-for-scale has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"50bdba9a945c69ff5ed2087ab5805766b9af93486dffbf3e9a30dacbe43e9e38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"5120549ea0a4e43a54af415fcf4738f68bf5643f4b2148f738ff9eceeccfaf0b","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00113a0c0\u003e: \n    resource group \"private-kas-hm9pdnmdszt5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-hm9pdnmdszt5\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f588ad3-c1af-45bb-9548-096867a64bbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f588ad3-c1af-45bb-9548-096867a64bbf\",\n      \"name\": \"2f588ad3-c1af-45bb-9548-096867a64bbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:29:27.633769413Z\",\n      \"endTime\": \"2026-09-30T17:53:32.876325702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0n6rovpqtd5op2d9vogdgft7tah1 still exists (deletion dispatched at 2026-09-30T17:29:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00113a0c0\u003e: \n    resource group \"private-kas-hm9pdnmdszt5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-hm9pdnmdszt5\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f588ad3-c1af-45bb-9548-096867a64bbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f588ad3-c1af-45bb-9548-096867a64bbf\",\n      \"name\": \"2f588ad3-c1af-45bb-9548-096867a64bbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:29:27.633769413Z\",\n      \"endTime\": \"2026-09-30T17:53:32.876325702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0n6rovpqtd5op2d9vogdgft7tah1 still exists (deletion dispatched at 2026-09-30T17:29:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a2f1c14b54bbf151c71f7611ce220c8f649fc58544dedc6741f9eb6cb082885","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"8da3c2a194a0976325b473c920b87ca0f08f429d6596d56f440a5f88b7289803","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:22:35Z\n  Ended:       2026-09-30T17:57:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446016-svc\", component=\"service-memory\", container=\"aro-hcp-frontend\", namespace=\"aro-hcp\", pod=\"aro-hcp-frontend-696bd4cfc8-xrrs2\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-frontend in pod aro-hcp-frontend-696bd4cfc8-xrrs2 (namespace aro-hcp) on cluster ci01-j5446016-svc has averaged 170% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:22:35Z\n  Ended:       2026-09-30T18:09:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446016-svc\", component=\"service-memory\", container=\"aro-hcp-frontend\", namespace=\"aro-hcp\", pod=\"aro-hcp-frontend-696bd4cfc8-bzvvh\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-frontend in pod aro-hcp-frontend-696bd4cfc8-bzvvh (namespace aro-hcp) on cluster ci01-j5446016-svc has averaged 209.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"6f2a3d24dea55f39c32293802a0bd966db6157d8bba5efef9a125e32ff239396","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"76a0103093e7faf35353100b194b7ab9cbc8dcbb3627657d6f21d44e6293a1fc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-622nv7gb94zh/cluster-candidate-4-22-jcthkg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c422a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-jcthkg: failed waiting for cluster=\"cluster-candidate-4-22-jcthkg\" in resourcegroup=\"rg-candidate-4-22-622nv7gb94zh\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\",\n      \"name\": \"6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:38.429049015Z\",\n      \"endTime\": \"2026-09-30T17:21:45.984146934Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 22m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-622nv7gb94zh/cluster-candidate-4-22-jcthkg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c422a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-jcthkg: failed waiting for cluster=\"cluster-candidate-4-22-jcthkg\" in resourcegroup=\"rg-candidate-4-22-622nv7gb94zh\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\",\n      \"name\": \"6c7ea5b6-e38d-4282-a8bd-b4e196c0a2b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:38.429049015Z\",\n      \"endTime\": \"2026-09-30T17:21:45.984146934Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 22m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"77acf5c6a4c1ea1ce868ef21009e33b703b2ce98332fc81071291752cff27bbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"fc6c0dff5250f75f2d3a3b3924e1b108f5097b463bf08d1d0cb2654087185cb1","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0029f60f0\u003e: \n    resource group \"complex-cilium-kv-vkmqwlbstl86\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vkmqwlbstl86\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758b0048-541b-41f7-b362-273a15532107\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758b0048-541b-41f7-b362-273a15532107\",\n      \"name\": \"758b0048-541b-41f7-b362-273a15532107\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:24:38.640589117Z\",\n      \"endTime\": \"2026-09-30T17:48:45.86210432Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1cult37rnvjnunif5m4d75jg68nc still exists (deletion dispatched at 2026-09-30T17:24:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0029f60f0\u003e: \n    resource group \"complex-cilium-kv-vkmqwlbstl86\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vkmqwlbstl86\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758b0048-541b-41f7-b362-273a15532107\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758b0048-541b-41f7-b362-273a15532107\",\n      \"name\": \"758b0048-541b-41f7-b362-273a15532107\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:24:38.640589117Z\",\n      \"endTime\": \"2026-09-30T17:48:45.86210432Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1cult37rnvjnunif5m4d75jg68nc still exists (deletion dispatched at 2026-09-30T17:24:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ee7a67480aaf5942b4fa1b7932429433c85d22360ab642c189c1e18356ec6a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"334ea0166a08010cff76f9d98df6bd32216ced385950ba9b5e9cd0a35c2140ea","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 93 time(s) (25 unknown, 68 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T17:04:15Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0e8dbaqbo8t5dt5vdcasi8k08a4q-v2v3z7b8u7q0n1w\", pod=\"kube-state-metrics-hcp-754f855b69-w9pxh\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0e8dbaqbo8t5dt5vdcasi8k08a4q-v2v3z7b8u7q0n1w/kube-state-metrics-hcp-754f855b69-w9pxh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:34:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"router-79bddd4695-4xzx4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/router-79bddd4695-4xzx4 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"multus-admission-controller-77fb5c8d8f-hmg8g\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/multus-admission-controller-77fb5c8d8f-hmg8g has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:34:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"router-6c4c4f9b46-9jc7h\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/router-6c4c4f9b46-9jc7h has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"network-node-identity-7b6f464c54-ttl5f\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/network-node-identity-7b6f464c54-ttl5f has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lqugf7p62g2vu15647hciuo7rqe-ephemeral-disk\", pod=\"kube-state-metrics-hcp-754f855b69-5lx7w\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lqugf7p62g2vu15647hciuo7rqe-ephemeral-disk/kube-state-metrics-hcp-754f855b69-5lx7w has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f1mqu517if66tcq7vv1883rm6svr2-x0m6x6k6h4c6v2j\", pod=\"kube-state-metrics-hcp-754f855b69-cvfnm\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f1mqu517if66tcq7vv1883rm6svr2-x0m6x6k6h4c6v2j/kube-state-metrics-hcp-754f855b69-cvfnm has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f19n96q5slf0m0lsru0ta7ea87dun-h5n4x4c7s1h6n9x\", pod=\"kube-state-metrics-hcp-754f855b69-5ds5m\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f19n96q5slf0m0lsru0ta7ea87dun-h5n4x4c7s1h6n9x/kube-state-metrics-hcp-754f855b69-5ds5m has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-30T17:05:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"ovnkube-control-plane-549bdb54d-n62hb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/ovnkube-control-plane-549bdb54d-n62hb has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-30T17:06:13Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0g0m8v1rghh660hh9lh81tcp6q5n-private-ingress\", pod=\"network-node-identity-5469fd6495-rbqgx\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0g0m8v1rghh660hh9lh81tcp6q5n-private-ingress/network-node-identity-5469fd6495-rbqgx has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-30T17:06:14Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"kube-controller-manager-59b5946b4c-8dnp4\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/kube-controller-manager-59b5946b4c-8dnp4 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-30T17:08:15Z\n  Ended:       2026-09-30T17:14:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0nnvckp9ok2ivvajoua5mi6rfcis-e4a2q0w1u4z3v6w\", pod=\"openshift-apiserver-5b6785f9dc-fc7tr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0nnvckp9ok2ivvajoua5mi6rfcis-e4a2q0w1u4z3v6w/openshift-apiserver-5b6785f9dc-fc7tr has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-30T17:08:15Z\n  Ended:       2026-09-30T17:14:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f15npkrjd452rtt7ar07bisihfskg-j2c7h4h2c6p4s3p\", pod=\"openshift-apiserver-7dd55877cd-gkcd5\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f15npkrjd452rtt7ar07bisihfskg-j2c7h4h2c6p4s3p/openshift-apiserver-7dd55877cd-gkcd5 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-30T17:08:15Z\n  Ended:       2026-09-30T17:14:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0uvlpj2ovpuf5b8grf3i9edvpodq-g3v6g1h5e0h5l2j\", pod=\"openshift-apiserver-758c4878d9-7j48d\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0uvlpj2ovpuf5b8grf3i9edvpodq-g3v6g1h5e0h5l2j/openshift-apiserver-758c4878d9-7j48d has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-30T17:11:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0g0m8v1rghh660hh9lh81tcp6q5n-private-ingress\", pod=\"network-node-identity-5469fd6495-f7zgd\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0g0m8v1rghh660hh9lh81tcp6q5n-private-ingress/network-node-identity-5469fd6495-f7zgd has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-09-30T17:11:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"router-8565cb6474-894zb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/router-8565cb6474-894zb has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-09-30T17:11:13Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"kube-apiserver-674f58879-j82fb\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/kube-apiserver-674f58879-j82fb has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-09-30T17:11:14Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"network-node-identity-7b6f464c54-gjfm6\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/network-node-identity-7b6f464c54-gjfm6 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-09-30T17:11:14Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d\", pod=\"openshift-apiserver-f89c544f9-jcfcs\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0lols5idrpgl73f9vshufog8tnob-e0j9d8p1c1m0o7d/openshift-apiserver-f89c544f9-jcfcs has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-09-30T17:11:14Z\n  Ended:       2026-09-30T17:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0suq7elir5mh0aa877skrjk5onf2-t7t6k6q0r6g9t5v\", pod=\"openshift-apiserver-5d4fc6f8c7-68m94\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0suq7elir5mh0aa877skrjk5onf2-t7t6k6q0r6g9t5v/openshift-apiserver-5d4fc6f8c7-68m94 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-09-30T17:11:14Z\n  Ended:       2026-09-30T17:27:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5f0suq7elir5mh0aa877skrjk5onf2-t7t6k6q0r6g9t5v\", pod=\"kube-apiserver-87c8b5fd9-ph74m\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5f0suq7elir5mh0aa877skrjk5onf2-t7t6k6q0r6g9t5v/kube-apiserver-87c8b5fd9-ph74m has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-09-30T17:13:16Z\n  Ended:       2026-09-30T17:22:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-p75kr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-p75kr has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:13:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-xfhsn\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-xfhsn has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:13:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5446016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"set-kubelet-parameters-for-scale-vdkwr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod kube-system/set-kubelet-parameters-for-scale-vdkwr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"817d70175a946670d824df2994758e55662874cbe6bffc0f86e9fdb75bdb8136","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"78aa4ea7e4450ab671d1d7955078133f07d134b1528dcc6a4a47bb2d342a0b2e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010c0210\u003e: \n    resource group \"private-keyvault-9g9ksdvdr2w9\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9g9ksdvdr2w9\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/135095ef-4130-48de-a890-36a3f2ec4ba2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/135095ef-4130-48de-a890-36a3f2ec4ba2\",\n      \"name\": \"135095ef-4130-48de-a890-36a3f2ec4ba2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:18:16.6672625Z\",\n      \"endTime\": \"2026-09-30T17:42:26.002126676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1cjpvuq75dcn2gn08v3gqss4ksup still exists (deletion dispatched at 2026-09-30T17:18:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010c0210\u003e: \n    resource group \"private-keyvault-9g9ksdvdr2w9\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9g9ksdvdr2w9\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/135095ef-4130-48de-a890-36a3f2ec4ba2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/135095ef-4130-48de-a890-36a3f2ec4ba2\",\n      \"name\": \"135095ef-4130-48de-a890-36a3f2ec4ba2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:18:16.6672625Z\",\n      \"endTime\": \"2026-09-30T17:42:26.002126676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1cjpvuq75dcn2gn08v3gqss4ksup still exists (deletion dispatched at 2026-09-30T17:18:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6299742b1ff6c5e532899640f78e385eadf28d949a1e95dfc697f72ee1a89ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"5fe0d75300377f26dad14459efb5218ed6d1b660024fd855d7728e47a4e9c833","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-m98nfkv66wgc/cluster-candidate-4-23-2zm5lz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013ea120\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-2zm5lz: failed waiting for cluster=\"cluster-candidate-4-23-2zm5lz\" in resourcegroup=\"rg-candidate-4-23-m98nfkv66wgc\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e5bac24-0540-4cfd-8079-7530283faa59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e5bac24-0540-4cfd-8079-7530283faa59\",\n      \"name\": \"7e5bac24-0540-4cfd-8079-7530283faa59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:39.678276861Z\",\n      \"endTime\": \"2026-09-30T17:21:45.946479348Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-m98nfkv66wgc/cluster-candidate-4-23-2zm5lz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013ea120\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-2zm5lz: failed waiting for cluster=\"cluster-candidate-4-23-2zm5lz\" in resourcegroup=\"rg-candidate-4-23-m98nfkv66wgc\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e5bac24-0540-4cfd-8079-7530283faa59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e5bac24-0540-4cfd-8079-7530283faa59\",\n      \"name\": \"7e5bac24-0540-4cfd-8079-7530283faa59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:39.678276861Z\",\n      \"endTime\": \"2026-09-30T17:21:45.946479348Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ab90cece596927f29bb9d77cee68680b6565fc35bcb0bb0f4102a61da6891b98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"alert","signature_id":"77cae9b2004bd675a604ffcac759cea6f79b1231e19f28fc49345d31d320e06a","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0m156ms43o6rvr33gnk6bjr3pfhe-a6a7t4q2j6v8c3t/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0lqhibjmokvvfkc1lvvbtckpvakc-y8w8x7w3j0c3f3n/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T18:04:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5446016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.68:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5446016/providers/microsoft.monitor/accounts/hcps-j5446016\", namespace=\"ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-5hwk9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t5f0jkpt55pcfu9tg9un71gt3tsm5a6-k9g8q9y9s6m6e2p/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"b1c51a936db26863920a8afdc1896743fd6da2b5b06d9d7623fc4d6469e5d2b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"6616b57a77390d9fcea017a145f401ec9a44bec8381030ff7da7d5b66342bea0","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0029c4180\u003e: \n    resource group \"feature-aggregation-bffzrcs9xcvk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-bffzrcs9xcvk\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2db4f57-9d20-454f-8d38-b79b992f2f32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2db4f57-9d20-454f-8d38-b79b992f2f32\",\n      \"name\": \"b2db4f57-9d20-454f-8d38-b79b992f2f32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:34:12.215869805Z\",\n      \"endTime\": \"2026-09-30T17:58:12.821045662Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1rm3k8s9324kfkruicq09u1qjvn0 still exists (deletion dispatched at 2026-09-30T17:34:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0029c4180\u003e: \n    resource group \"feature-aggregation-bffzrcs9xcvk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-bffzrcs9xcvk\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2db4f57-9d20-454f-8d38-b79b992f2f32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2db4f57-9d20-454f-8d38-b79b992f2f32\",\n      \"name\": \"b2db4f57-9d20-454f-8d38-b79b992f2f32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:34:12.215869805Z\",\n      \"endTime\": \"2026-09-30T17:58:12.821045662Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f1rm3k8s9324kfkruicq09u1qjvn0 still exists (deletion dispatched at 2026-09-30T17:34:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dec046171ebe607223e3946d8ec1e31bae7db85e40dc9cfdf5b3ea1927a673f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"c47e9e92a6f4e295102296fad094562a1d04cf7aee29b799aa62c27f9514ae5c","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1a380\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-rwh7s9rvp89r\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\",\n      \"name\": \"eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:15.764760069Z\",\n      \"endTime\": \"2026-09-30T17:21:16.086415783Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m59s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1a380\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-rwh7s9rvp89r\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\",\n      \"name\": \"eb690e90-f4f6-4eb4-9d4f-fe046ccd3d44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:15.764760069Z\",\n      \"endTime\": \"2026-09-30T17:21:16.086415783Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m59s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e8724405a76b7347f766f06c463ec97e9d46716b0d7dc84ffa79d86931570487","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"9ebe0a8379936d0120b8baffd7195e8f84ec0ac55ddd2ce57a2b08da074be15e","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0001aacc0\u003e: \n    resource group \"rg-hs-pre--r48mqt9lrw4s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-fqsxq9\" in resourcegroup=\"rg-hs-pre--r48mqt9lrw4s\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c44789-52a8-49c2-b053-4835c3571170\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c44789-52a8-49c2-b053-4835c3571170\",\n      \"name\": \"49c44789-52a8-49c2-b053-4835c3571170\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:16:19.615591526Z\",\n      \"endTime\": \"2026-09-30T17:40:25.981909891Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0hjlm2j48ooq92uj0su5rbnsq03t still exists (deletion dispatched at 2026-09-30T17:16:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0001aacc0\u003e: \n    resource group \"rg-hs-pre--r48mqt9lrw4s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-fqsxq9\" in resourcegroup=\"rg-hs-pre--r48mqt9lrw4s\" to finish deleting: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c44789-52a8-49c2-b053-4835c3571170\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49c44789-52a8-49c2-b053-4835c3571170\",\n      \"name\": \"49c44789-52a8-49c2-b053-4835c3571170\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:16:19.615591526Z\",\n      \"endTime\": \"2026-09-30T17:40:25.981909891Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5f0hjlm2j48ooq92uj0su5rbnsq03t still exists (deletion dispatched at 2026-09-30T17:16:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f315cd50d7b3b94dd0b6a695aef35c48e56ac58791ce91bcaebd9aebff7e3076","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105327789455446016","occurred_at":"2026-09-30T16:03:39Z","failed_at":"e2e","signature_id":"335ab959219cb206e30234fc1cf72ca10d7c54d0f8706508b7269b3ad089b86d","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a8280\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ztrwgb4tgx4r\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\",\n      \"name\": \"fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:39.354643704Z\",\n      \"endTime\": \"2026-09-30T17:21:45.995494198Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m45s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a8280\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-ztrwgb4tgx4r\" to finish creating: GET https://rp.j5446016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\",\n      \"name\": \"fb9dfc1a-3b4b-487f-b91c-0f6b1985bf0d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:52:39.354643704Z\",\n      \"endTime\": \"2026-09-30T17:21:45.995494198Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m45s ago\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":23,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":23,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105322513406889984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7254,"pr_state":"closed","pr_sha":"fed46bdb633e2219c0b9e04a9984e1a83a49fd45","final_merged_sha":"fed46bdb633e2219c0b9e04a9984e1a83a49fd45","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T15:42:56Z","started_at":"2026-09-30T15:42:56Z","completed_at":"2026-09-30T18:13:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:29:07.618773939Z","timing_metadata_checked_at":"2026-09-30T18:29:07.618773939Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:29:07.621380532Z","region_metadata_checked_at":"2026-09-30T18:29:07.621380532Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105320779594862592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7242,"pr_state":"closed","pr_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","final_merged_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T15:36:03Z","started_at":"2026-09-30T15:36:03Z","completed_at":"2026-09-30T18:00:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:08:58.133101876Z","timing_metadata_checked_at":"2026-09-30T18:08:58.133101876Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:08:58.135892768Z","region_metadata_checked_at":"2026-09-30T18:08:58.135892768Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6e4cb3f085907f3c3864d931651ab5669b26a959dec6014f8a1ddf72ce999234","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105320779594862592","occurred_at":"2026-09-30T15:36:03Z","failed_at":"e2e","signature_id":"c1bac4e92db70f51e3a0c22de77dcee2f5a3f262e7aa5d1b8931dc415680d6be","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca3800\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xw6tpr5d2lsl\" to finish creating: GET https://rp.j4862592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\",\n      \"name\": \"3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:03:28.253892004Z\",\n      \"endTime\": \"2026-09-30T17:22:36.043787859Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-h2g79-bb4np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xw6tpr5d2lsl--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-h2g79-bb4np to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-h2g79-bb4np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xw6tpr5d2lsl--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-h2g79-bb4np to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca3800\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xw6tpr5d2lsl\" to finish creating: GET https://rp.j4862592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\",\n      \"name\": \"3d2602fc-b3fa-43e1-b8e3-5be7f209f5fc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:03:28.253892004Z\",\n      \"endTime\": \"2026-09-30T17:22:36.043787859Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-h2g79-bb4np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xw6tpr5d2lsl--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-h2g79-bb4np to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-h2g79-bb4np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-xw6tpr5d2lsl--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-h2g79-bb4np to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105320628276957184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"8719a8a13bd69122b6b8841f4e14e467b41e9993","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T15:35:27Z","started_at":"2026-09-30T15:35:27Z","completed_at":"2026-09-30T18:05:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:08:55.716111282Z","timing_metadata_checked_at":"2026-09-30T18:08:55.716111282Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:08:55.719478573Z","region_metadata_checked_at":"2026-09-30T18:08:55.719478573Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6967/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105320296662700032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6967,"pr_state":"closed","pr_sha":"3a81d064b855c5a0862d9de651db3428e2519ace","final_merged_sha":"3a81d064b855c5a0862d9de651db3428e2519ace","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T15:34:08Z","started_at":"2026-09-30T15:34:08Z","completed_at":"2026-09-30T17:58:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:08:57.124022374Z","timing_metadata_checked_at":"2026-09-30T18:08:57.124022374Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:08:57.126950066Z","region_metadata_checked_at":"2026-09-30T18:08:57.126950066Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105310240105828352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"e01d7004aa224ff873a1a813ac3e6932792e1135","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T14:54:10Z","started_at":"2026-09-30T14:54:10Z","completed_at":"2026-09-30T17:24:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T17:36:59.989528706Z","timing_metadata_checked_at":"2026-09-30T17:36:59.989528706Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T17:36:59.992520399Z","region_metadata_checked_at":"2026-09-30T17:36:59.992520399Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8e7c8ad5f90ec5a260a9d8f0dd5acbd6718f58b843a38edcfddaffb9624db754","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105310240105828352","occurred_at":"2026-09-30T14:54:10Z","failed_at":"alert","signature_id":"14d730d688feed9879dfb08b05aec4624d9b99fa53a65137ea56efd900ad0e9d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T16:05:30Z\n  Ended:       2026-09-30T16:33:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5828352-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5e65ks30us7u7a3qg1vdbknb1vvqs6-t3z0s0t0v4e0p5i\", pod=\"router-865f6cd877-rnq4k\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5e65ks30us7u7a3qg1vdbknb1vvqs6-t3z0s0t0v4e0p5i/router-865f6cd877-rnq4k has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T16:07:32Z\n  Ended:       2026-09-30T16:25:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5828352-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5e65gj37b763s4sdv3kievh5fge8ub-c7d3y8f1x8e6t7o\", pod=\"router-8f95b5bf6-pw2hq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5e65gj37b763s4sdv3kievh5fge8ub-c7d3y8f1x8e6t7o/router-8f95b5bf6-pw2hq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"a5901e8e124043896aab06ceb2f5b4dcffc9346624ce0b8afa494bbc23edb05e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105310240105828352","occurred_at":"2026-09-30T14:54:10Z","failed_at":"e2e","signature_id":"55c7211b6b6d239408f79dea1df4f8eea0fbc4889550177ece4353034cb083f3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-xnh68bcrp77f/cluster-candidate-4-21-944b8p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000874f60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-944b8p: failed waiting for cluster=\"cluster-candidate-4-21-944b8p\" in resourcegroup=\"rg-candidate-4-21-xnh68bcrp77f\" to finish creating: GET https://rp.j5828352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0796d72b-193f-46f8-9e78-aaf27448038b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0796d72b-193f-46f8-9e78-aaf27448038b\",\n      \"name\": \"0796d72b-193f-46f8-9e78-aaf27448038b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T15:56:05.509326654Z\",\n      \"endTime\": \"2026-09-30T16:25:07.850265278Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-xnh68bcrp77f/cluster-candidate-4-21-944b8p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000874f60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-944b8p: failed waiting for cluster=\"cluster-candidate-4-21-944b8p\" in resourcegroup=\"rg-candidate-4-21-xnh68bcrp77f\" to finish creating: GET https://rp.j5828352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0796d72b-193f-46f8-9e78-aaf27448038b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0796d72b-193f-46f8-9e78-aaf27448038b\",\n      \"name\": \"0796d72b-193f-46f8-9e78-aaf27448038b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T15:56:05.509326654Z\",\n      \"endTime\": \"2026-09-30T16:25:07.850265278Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105308204090003456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"cdf5413a948fe5306876c42a99a496f2a0c8e352","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T14:46:05Z","started_at":"2026-09-30T14:46:05Z","completed_at":"2026-09-30T17:18:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T17:36:59.960296675Z","timing_metadata_checked_at":"2026-09-30T17:36:59.960296675Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T17:36:59.966010562Z","region_metadata_checked_at":"2026-09-30T17:36:59.966010562Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105298021888561152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"7551c0170c5cffea9e64f3707dcf2378a40b7d3f","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T14:05:37Z","started_at":"2026-09-30T14:05:37Z","completed_at":"2026-09-30T16:47:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T17:05:27.830030213Z","timing_metadata_checked_at":"2026-09-30T17:05:27.830030213Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T17:05:27.8340818Z","region_metadata_checked_at":"2026-09-30T17:05:27.8340818Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b8ebdf0940900016061394ed2ad755d1660f7cee2cebb7def0a9cfdda9f40918","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105298021888561152","occurred_at":"2026-09-30T14:05:37Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T15:25:46Z\n  Ended:       2026-09-30T15:31:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8561152-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105297136064139264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T14:02:06Z","started_at":"2026-09-30T14:02:06Z","completed_at":"2026-09-30T16:47:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T17:05:28.238065946Z","timing_metadata_checked_at":"2026-09-30T17:05:28.238065946Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T17:05:28.241049837Z","region_metadata_checked_at":"2026-09-30T17:05:28.241049837Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9fd7cfdd1f1fe96b8503a67f56e20510aff7491ca9c6c32c89349b13e11c8b24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105297136064139264","occurred_at":"2026-09-30T14:02:06Z","failed_at":"alert","signature_id":"632e48363b16662cf206e035c9d5483425dd46982f9be6da0c90dbb11133e9ff","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T15:21:57Z\n  Ended:       2026-09-30T15:50:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4139264-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5dgduvrdd5glf5k0m6vesv46nt0jhm-private-kas\", pod=\"router-55bfdfc49b-qhf45\", region=\"westus3\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5dgduvrdd5glf5k0m6vesv46nt0jhm-private-kas/router-55bfdfc49b-qhf45 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"be46e56c2bfaa5f6ebe5edb62bcb1125fffb033614a679ece26327f52a7d6553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105297136064139264","occurred_at":"2026-09-30T14:02:06Z","failed_at":"e2e","signature_id":"5cbfd891b17369ec915b5a99867c4b2f1c6fc13517bd9b12c29a9682a6a58e0a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001f61c0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-2ggb82wh5r22\" to finish creating: GET https://rp.j4139264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50346f69-92e6-45cd-a218-e9b1ff541e03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50346f69-92e6-45cd-a218-e9b1ff541e03\",\n      \"name\": \"50346f69-92e6-45cd-a218-e9b1ff541e03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T15:09:42.512906803Z\",\n      \"endTime\": \"2026-09-30T15:38:51.371261651Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001f61c0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-2ggb82wh5r22\" to finish creating: GET https://rp.j4139264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50346f69-92e6-45cd-a218-e9b1ff541e03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50346f69-92e6-45cd-a218-e9b1ff541e03\",\n      \"name\": \"50346f69-92e6-45cd-a218-e9b1ff541e03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T15:09:42.512906803Z\",\n      \"endTime\": \"2026-09-30T15:38:51.371261651Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105295109229645824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6944,"pr_state":"open","pr_sha":"945b5ce92c4d576dbad252aa980a2a25b088b764","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T13:54:03Z","started_at":"2026-09-30T13:54:03Z","completed_at":"2026-09-30T16:20:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:33:52.24692486Z","timing_metadata_checked_at":"2026-09-30T16:33:52.24692486Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:33:52.250565749Z","region_metadata_checked_at":"2026-09-30T16:33:52.250565749Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0136923f00aa5803e69549b2f7c0b328d8673edf7206e3f483ddd8c0275b8653","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105295109229645824","occurred_at":"2026-09-30T13:54:03Z","failed_at":"alert","signature_id":"ae184cd60d96c8302bc4346af9b111ff0bed8c1839b502ea23adb68311ddbd47","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T15:24:36Z\n  Ended:       2026-09-30T15:30:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9645824-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5d7bu25qfk2cfhnp3n9qlq37dgcjbd-private-ingress\", pod=\"etcd-0\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5d7bu25qfk2cfhnp3n9qlq37dgcjbd-private-ingress/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e0056126b95491bf1abddd1ed55a47da671b413bb44e9883287392b1bf030a8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105295109229645824","occurred_at":"2026-09-30T13:54:03Z","failed_at":"e2e","signature_id":"842ff61370b596e7c3d23fa920da25dad023b70b10d4ba2b38239c18c65c13b1","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c9ee28\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c9ee28\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"closed","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"273677ce69aa7a8dc7e1da98b8942d3ea48b24d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T13:51:53Z","started_at":"2026-09-30T13:51:53Z","completed_at":"2026-09-30T15:13:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T15:30:36.77474978Z","timing_metadata_checked_at":"2026-09-30T15:30:36.77474978Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T15:30:36.77796887Z","region_metadata_checked_at":"2026-09-30T15:30:36.77796887Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"2f1dd9c1d7339dfb953eaed1a62cc085d124194a724ea931403eca39d85fab87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","occurred_at":"2026-09-30T13:51:53Z","failed_at":"provision","signature_id":"116a812adf4da3f0b766ca7f0e970c4b7b28de7fba2c7153f753459f2b443b8c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T14:35:13.661Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T14:35:14.082Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j1358720-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j1358720-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e962c7b1-5eb5-497b-9341-4b3a121f354e region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e962c7b1-5eb5-497b-9341-4b3a121f354e INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T14:35:14.083Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:35:14.275Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T14:35:14.491Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T14:35:14.491Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:35:14.524Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:45:17.368Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T14:45:17.369Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T14:46:17.370Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T14:46:17.815Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j1358720-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j1358720-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e962c7b1-5eb5-497b-9341-4b3a121f354e region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e962c7b1-5eb5-497b-9341-4b3a121f354e INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T14:46:17.815Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:18.042Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T14:46:18.350Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:18.350Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:18.601Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T14:46:19.301Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:19.301Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:19.487Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:19.912Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T14:46:20.506Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T14:46:20.869Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:21.189Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:21.550Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:46:21.905Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T14:46:21.905Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:46:21.951Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T14:56:23.257Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T14:56:23.257Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T14:57:23.259Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T14:57:24.108Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j1358720-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j1358720-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e962c7b1-5eb5-497b-9341-4b3a121f354e region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e962c7b1-5eb5-497b-9341-4b3a121f354e INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T14:57:24.108Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:24.411Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T14:57:24.749Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:24.749Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:25.002Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T14:57:25.620Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:25.620Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:25.977Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:26.226Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T14:57:26.765Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T14:57:26.974Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:27.201Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:27.763Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T14:57:28.017Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T14:57:28.017Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T14:57:28.079Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T15:05:13.695Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-30T15:05:13.695Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"a3c608f908175e627dcf21ec2ce22918017d607cbfdd8561892810dabaf06d9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","occurred_at":"2026-09-30T13:51:53Z","failed_at":"alert","signature_id":"6ab116244e8b6c55af7310ca393f986f1be56582f51b8e5b67d25e76cd3995b9","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:52:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.42:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1358720/providers/microsoft.monitor/accounts/services-j1358720\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-r6958\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:52:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.42:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1358720/providers/microsoft.monitor/accounts/services-j1358720\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-r6958\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"b044b6e7944e4c2d0ee8033dfbfc75317825839f2636b750e6bfd3db8dd63219","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","occurred_at":"2026-09-30T13:51:53Z","failed_at":"alert","signature_id":"b25416717b76880c6609e81cf02b1d01fc79601f025ad76a9d31a41cb5607615","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:42:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-png7s\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-png7s has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:43:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-bzbm7\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-bzbm7 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:46:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1358720-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-mkkjw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-mkkjw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"e472ccfbd946897ea421439e8a17caab6fd959a80bbedf5930c0ed20d8216866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294563651358720","occurred_at":"2026-09-30T13:51:53Z","failed_at":"alert","signature_id":"8d542cbb8d9ed9afab5c0b08d7fb2e462252dfaa5e9d052f0e6a6910d7b32fe6","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T14:51:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j1358720-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j1358720-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294515421057024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"eb16c2e94e7f2596a478e5570733fb66ffcca02f","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T13:51:41Z","started_at":"2026-09-30T13:51:41Z","completed_at":"2026-09-30T14:15:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:27:20.627331401Z","timing_metadata_checked_at":"2026-09-30T14:27:20.627331401Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-30T14:27:20.63036099Z","region_metadata_checked_at":"2026-09-30T15:30:38.135687941Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e2f618c09f9702822e92ca3e057c1fb271128da0e59eeb7a1a3ece0340469634","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294515421057024","occurred_at":"2026-09-30T13:51:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294443023175680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"closed","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"c11d1aab795ebbd762ddef1d8188a105d875d002","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T13:51:24Z","started_at":"2026-09-30T13:51:24Z","completed_at":"2026-09-30T16:22:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:33:52.419994008Z","timing_metadata_checked_at":"2026-09-30T16:33:52.419994008Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:33:52.423155698Z","region_metadata_checked_at":"2026-09-30T16:33:52.423155698Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294127649263616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7269,"pr_state":"closed","pr_sha":"ba33a85d4cb42b48ca7efbabd149d16d8e41c231","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T13:49:57Z","started_at":"2026-09-30T13:49:57Z","completed_at":"2026-09-30T16:11:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:23:15.939698075Z","timing_metadata_checked_at":"2026-09-30T16:23:15.939698075Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:23:15.942871665Z","region_metadata_checked_at":"2026-09-30T16:23:15.942871665Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"5188a67ac55429a9dadf31e3ef3b6bcc3649ff5d5b62e41fd496aed8ce8f6bf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294127649263616","occurred_at":"2026-09-30T13:49:57Z","failed_at":"e2e","signature_id":"04e00d32fdeba2bfdaf45c1fa9c7407287ff098c212502b818e7b2573e85be65","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:242]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4a880\u003e: \n    failed starting cluster creation \"acr-pull-create\" in resourcegroup=\"acr-pull-create-v5r8zv9r2jvq\": PUT https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-v5r8zv9r2jvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/acr-pull-create\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-v5r8zv9r2jvq/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-mi\\\": containerRegistry requires cluster version 4.22.0 or above\",\n        \"target\": \"properties.platform.containerRegistry.managedIdentity\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:242]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4a880\u003e: \n    failed starting cluster creation \"acr-pull-create\" in resourcegroup=\"acr-pull-create-v5r8zv9r2jvq\": PUT https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-v5r8zv9r2jvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/acr-pull-create\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-v5r8zv9r2jvq/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-mi\\\": containerRegistry requires cluster version 4.22.0 or above\",\n        \"target\": \"properties.platform.containerRegistry.managedIdentity\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e648505864d678d9f50fe37f530cf1b1dab08c26dec37f3ae5734914d2656ab9","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-pull-mi\": containerRegistry requires cluster version 4.22.0 or above; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7b4b85545a2724ae2c05a95fbaa529e63e48b14e3ffac9bd1cce64cc6bb215a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294127649263616","occurred_at":"2026-09-30T13:49:57Z","failed_at":"e2e","signature_id":"d6cef5f759144348562c7441abc3cf578b5ca595385f45cabf910f3ccade4227","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:96]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c48700\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-7mqkjzxbfvhm\": PUT https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-7mqkjzxbfvhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"Private\\\": not supported when customerProperties.version.id is below 4.22\",\n        \"target\": \"properties.api.visibility\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:96]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c48700\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-7mqkjzxbfvhm\": PUT https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-7mqkjzxbfvhm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"Private\\\": not supported when customerProperties.version.id is below 4.22\",\n        \"target\": \"properties.api.visibility\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"082efb8c79c4bc42f53051748bec5dc917c7daff5a16459440cdcba2f63beaf8","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"Private\": not supported when customerProperties.version.id is below 4.22; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f413f40c2294cef62269313dc811d51dd9567628cdb3c9503e66686227fa935c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7269/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105294127649263616","occurred_at":"2026-09-30T13:49:57Z","failed_at":"e2e","signature_id":"6693a3c38bba6bece1046d71a702c519b29e94b4f42ba132e9c2c5a8c666092f","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:191]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ece360\u003e: \n    PATCH https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-s7rqrhtvbcff/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/kms-key-rotate-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"484184ba23584b37813d670c444e552b\\\": KMS key version rotation requires cluster version 4.22.0 or above\",\n        \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.activeKey.version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:191]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ece360\u003e: \n    PATCH https://rp.j9263616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-s7rqrhtvbcff/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/kms-key-rotate-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"484184ba23584b37813d670c444e552b\\\": KMS key version rotation requires cluster version 4.22.0 or above\",\n        \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.activeKey.version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fdef26532b23d7ffbf4352a28e772bf3eb538e6b3ef837cff2ecfc32992615a2","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003chex\u003e\": KMS key version rotation requires cluster version 4.22.0 or above; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["082efb8c79c4bc42f53051748bec5dc917c7daff5a16459440cdcba2f63beaf8","e648505864d678d9f50fe37f530cf1b1dab08c26dec37f3ae5734914d2656ab9","fdef26532b23d7ffbf4352a28e772bf3eb538e6b3ef837cff2ecfc32992615a2"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105293545626669056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7268,"pr_state":"open","pr_sha":"035752c2d1ac4dfa83de437669f06c1bbd17fecb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T13:47:36Z","started_at":"2026-09-30T13:47:36Z","completed_at":"2026-09-30T16:19:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:33:52.645926887Z","timing_metadata_checked_at":"2026-09-30T16:33:52.645926887Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:33:52.651686868Z","region_metadata_checked_at":"2026-09-30T16:33:52.651686868Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7246/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105289648665268224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7246,"pr_state":"closed","pr_sha":"26be4d462d878ae28865cbe5d86888fee258acfb","final_merged_sha":"26be4d462d878ae28865cbe5d86888fee258acfb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T13:32:21Z","started_at":"2026-09-30T13:32:21Z","completed_at":"2026-09-30T15:48:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T16:02:49.763342381Z","timing_metadata_checked_at":"2026-09-30T16:02:49.763342381Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T16:02:49.766475871Z","region_metadata_checked_at":"2026-09-30T16:02:49.766475871Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105279807200694272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"8470e4a8152fec1ad8e98ecdb32250dd598096a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T12:53:14Z","started_at":"2026-09-30T12:53:14Z","completed_at":"2026-09-30T15:14:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T15:30:38.051011611Z","timing_metadata_checked_at":"2026-09-30T15:30:38.051011611Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T15:30:38.0544819Z","region_metadata_checked_at":"2026-09-30T15:30:38.0544819Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"7a4c656e98cd11f37645f828fbef44146a64ed5a6d9807f5845edc915a3be75f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105279807200694272","occurred_at":"2026-09-30T12:53:14Z","failed_at":"e2e","signature_id":"d84fbb4f91a7ca89a512a9ccaac84a33ddc34058ff07b5aaf957fe0df53b120b","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea00e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-7mqggzhqq55s\" to finish creating: GET https://rp.j0694272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ec169ee-a122-45a3-a622-b7083455fb90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ec169ee-a122-45a3-a622-b7083455fb90\",\n      \"name\": \"1ec169ee-a122-45a3-a622-b7083455fb90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T14:16:43.796681639Z\",\n      \"endTime\": \"2026-09-30T14:35:52.497968724Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5o8a1j8f7l2m2y-np-1-qdcxj-hwbpg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5o8a1j8f7l2m2y-np-1-qdcxj-hwbpg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea00e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-7mqggzhqq55s\" to finish creating: GET https://rp.j0694272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ec169ee-a122-45a3-a622-b7083455fb90\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ec169ee-a122-45a3-a622-b7083455fb90\",\n      \"name\": \"1ec169ee-a122-45a3-a622-b7083455fb90\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T14:16:43.796681639Z\",\n      \"endTime\": \"2026-09-30T14:35:52.497968724Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5o8a1j8f7l2m2y-np-1-qdcxj-hwbpg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5o8a1j8f7l2m2y-np-1-qdcxj-hwbpg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fbcfa983645c3e2633f472d4edb029f4fb86ade620ae9a86f76fa5ddd9dc3a24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105279807200694272","occurred_at":"2026-09-30T12:53:14Z","failed_at":"e2e","signature_id":"b67af1235e148ac25954a38fdd4d5236c882e2e12c2417baba9e32624257a663","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000686fa0\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-hqw2kg\" in resourcegroup=\"rg-np-downgrade-hqw2kg-tzshqkp8zjqk\" to finish creating: GET https://rp.j0694272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\",\n      \"name\": \"0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T14:18:31.838214826Z\",\n      \"endTime\": \"2026-09-30T14:37:32.428641421Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hqw2kg-tzshqkp8zjqk-np-dg-hqw2kg--managed/providers/Microsoft.Compute/virtualMachines/h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hqw2kg-tzshqkp8zjqk-np-dg-hqw2kg--managed/providers/Microsoft.Compute/virtualMachines/h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000686fa0\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-hqw2kg\" in resourcegroup=\"rg-np-downgrade-hqw2kg-tzshqkp8zjqk\" to finish creating: GET https://rp.j0694272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\",\n      \"name\": \"0c88c27e-4a0d-4fb4-a4af-b21df9d3ccdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T14:18:31.838214826Z\",\n      \"endTime\": \"2026-09-30T14:37:32.428641421Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hqw2kg-tzshqkp8zjqk-np-dg-hqw2kg--managed/providers/Microsoft.Compute/virtualMachines/h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hqw2kg-tzshqkp8zjqk-np-dg-hqw2kg--managed/providers/Microsoft.Compute/virtualMachines/h2z3j3d0p8e5c3v-npdg-4-21-7x86w-jpqjj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105270530377519104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T12:16:23Z","started_at":"2026-09-30T12:16:23Z","completed_at":"2026-09-30T14:47:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:58:38.59964585Z","timing_metadata_checked_at":"2026-09-30T14:58:38.59964585Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:58:38.60287854Z","region_metadata_checked_at":"2026-09-30T14:58:38.60287854Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7260/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105265973241057280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7260,"pr_state":"closed","pr_sha":"d2063f84c5105758d4ff2370d77bd0c238c14b4d","final_merged_sha":"e2029a06a51da6cdf2ed5b5ef1c620dcc9cda152","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T11:58:16Z","started_at":"2026-09-30T11:58:16Z","completed_at":"2026-09-30T14:35:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:48:04.941795603Z","timing_metadata_checked_at":"2026-09-30T14:48:04.941795603Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:48:04.944855194Z","region_metadata_checked_at":"2026-09-30T14:48:04.944855194Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105265916785725440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7158,"pr_state":"open","pr_sha":"0ca8c9c41a80a8dbda691ac0542184fa1ebc5996","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T11:58:03Z","started_at":"2026-09-30T11:58:03Z","completed_at":"2026-09-30T14:21:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:38:02.18671305Z","timing_metadata_checked_at":"2026-09-30T14:38:02.18671305Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:38:02.194917923Z","region_metadata_checked_at":"2026-09-30T14:38:02.194917923Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7263/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105257717114343424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7263,"pr_state":"closed","pr_sha":"8481983b584ac7681861ebe6510e693bba610634","final_merged_sha":"8481983b584ac7681861ebe6510e693bba610634","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T11:25:28Z","started_at":"2026-09-30T11:25:28Z","completed_at":"2026-09-30T13:43:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T13:55:37.501328185Z","timing_metadata_checked_at":"2026-09-30T13:55:37.501328185Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T13:55:37.504463077Z","region_metadata_checked_at":"2026-09-30T13:55:37.504463077Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105255814120869888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"open","pr_sha":"834e3a4bfb7f68837a296014f2446af7c338f3b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T11:17:54Z","started_at":"2026-09-30T11:17:54Z","completed_at":"2026-09-30T13:54:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:06:22.890288049Z","timing_metadata_checked_at":"2026-09-30T14:06:22.890288049Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:06:22.893614938Z","region_metadata_checked_at":"2026-09-30T14:06:22.893614938Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6141,"pr_state":"open","pr_sha":"a0b9d9de5a50a5434f1e58fba07352436371588c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T11:09:49Z","started_at":"2026-09-30T11:09:49Z","completed_at":"2026-09-30T13:26:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T13:35:24.759114007Z","timing_metadata_checked_at":"2026-09-30T13:35:24.759114007Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T13:35:24.761907896Z","region_metadata_checked_at":"2026-09-30T13:35:24.761907896Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"20384cbfa72afe9dff06179fe6fb87433c2c519fd027d233b59ba3773ca7f40e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"a7d987a6289e47bbd6f06053a8e44ff6a83f6ca7a82baaa9d31d2b45509ddaa3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-889bmn\" for cluster \"cluster-candidate-4-20-889bmn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015841e0\u003e: \n    failed to create NodePool np-889bmn: failed waiting for nodepool=\"np-889bmn\" for cluster \"cluster-candidate-4-20-889bmn\" in resourcegroup=\"rg-candidate-4-20-qcvff5h4wj94\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e30a66c5-dd1a-49f7-a04b-0cec3c58378a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e30a66c5-dd1a-49f7-a04b-0cec3c58378a\",\n      \"name\": \"e30a66c5-dd1a-49f7-a04b-0cec3c58378a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:19:11.883646471Z\",\n      \"endTime\": \"2026-09-30T12:38:11.728721311Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-889bmn\" for cluster \"cluster-candidate-4-20-889bmn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015841e0\u003e: \n    failed to create NodePool np-889bmn: failed waiting for nodepool=\"np-889bmn\" for cluster \"cluster-candidate-4-20-889bmn\" in resourcegroup=\"rg-candidate-4-20-qcvff5h4wj94\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e30a66c5-dd1a-49f7-a04b-0cec3c58378a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e30a66c5-dd1a-49f7-a04b-0cec3c58378a\",\n      \"name\": \"e30a66c5-dd1a-49f7-a04b-0cec3c58378a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:19:11.883646471Z\",\n      \"endTime\": \"2026-09-30T12:38:11.728721311Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"25064b267613d8f703d629d9b485be5bba78d318d63590a4ace739fc27ee79e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade"}},{"row_id":"336db8d1f67cf62217ce7d703879cf57b593810d1a71ec5f9f4769100f10e56c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"814a3d9ab08f714d33b125c7043504f54189a3a864c0198073bdb76d3a178e45","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c681a0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-fl5kpw\" in resourcegroup=\"rg-np-version-upgrade-fl5kpw-96mvvhr695cd\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\",\n      \"name\": \"a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:36:12.039132485Z\",\n      \"endTime\": \"2026-09-30T12:55:21.071689427Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c681a0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-fl5kpw\" in resourcegroup=\"rg-np-version-upgrade-fl5kpw-96mvvhr695cd\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\",\n      \"name\": \"a0ac539a-44de-4d6b-be03-d9b6fc3ccdc0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:36:12.039132485Z\",\n      \"endTime\": \"2026-09-30T12:55:21.071689427Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7bbcc1c13014d006c1b016210955f659362471edd09845dfbd58f7a5de47ca40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"48929c5f392dda19c39d764a9d4117d26ea3b70f835a97462a0249c92cc9fdf4","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n    \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","failure_pattern":"expected all nodes to have label 'key1=value1'","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'"}},{"row_id":"89da932e4fb012a5fe65dd67c085f4252dcdbb87a417f92e8ad888ac4fcf01a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"95c89ae0f5cb2e646984ea99ad7d4214bf8a3269adb7da5693215c8731baef87","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000abb5a0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dr2mph\" in resourcegroup=\"rg-np-noedge-dr2mph-j7ng55t8mcf6\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78b330e2-3532-4f23-91f1-b24676955611\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78b330e2-3532-4f23-91f1-b24676955611\",\n      \"name\": \"78b330e2-3532-4f23-91f1-b24676955611\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:38:52.851263491Z\",\n      \"endTime\": \"2026-09-30T12:58:01.083392787Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000abb5a0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-dr2mph\" in resourcegroup=\"rg-np-noedge-dr2mph-j7ng55t8mcf6\" to finish creating: GET https://rp.j4354560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78b330e2-3532-4f23-91f1-b24676955611\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/78b330e2-3532-4f23-91f1-b24676955611\",\n      \"name\": \"78b330e2-3532-4f23-91f1-b24676955611\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:38:52.851263491Z\",\n      \"endTime\": \"2026-09-30T12:58:01.083392787Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9c90276c83c8f822e869c51bc43127253d3c5481f1f65631c683f33d3339a366","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"a27dbddc73d079caee9d5d24a8d5a0542aeb9a203f7e6c456afcbb92c10199e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:476]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:476]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"cd6a8919ca914c56894a6805380f22429dedc6fa6cf6b048342b1c85b7652ed6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"ea5287bbb6f52fd8ce7c07f23459d62fe096fd8dd852dfadcfb9dc8719d6e333","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"2517a1d16eaac19c98fc4a1983465611e0830eab2920dd83c5c7ac7bc707e3dd","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00082e468\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00082e468\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"ef24a3c54e88408e381821269c56e6921f93406266d98cb5a270ffad06a61906","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105253779644354560","occurred_at":"2026-09-30T11:09:49Z","failed_at":"e2e","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n    \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105244373301071872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"7551c0170c5cffea9e64f3707dcf2378a40b7d3f","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T10:32:26Z","started_at":"2026-09-30T10:32:26Z","completed_at":"2026-09-30T12:50:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:54:00.082357196Z","timing_metadata_checked_at":"2026-09-30T12:54:00.082357196Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:54:00.085735685Z","region_metadata_checked_at":"2026-09-30T12:54:00.085735685Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"73f8445ae422ae45e21aa18ed73ce4f5a4dcaf49bcc23bf952862dd41761c21f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105244373301071872","occurred_at":"2026-09-30T10:32:26Z","failed_at":"e2e","signature_id":"02b30f07b1850865809f15a839e63f1edb02365f906e6f1394faa96d6cedcc30","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wsv85m\" for cluster \"cluster-candidate-4-23-wsv85m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7d6c0\u003e: \n    failed to create NodePool np-wsv85m: failed waiting for nodepool=\"np-wsv85m\" for cluster \"cluster-candidate-4-23-wsv85m\" in resourcegroup=\"rg-candidate-4-23-x5s4hpmkwzhp\" to finish creating: GET https://rp.j1071872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/edf97b61-1366-48f5-936f-5378dbccb550\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/edf97b61-1366-48f5-936f-5378dbccb550\",\n      \"name\": \"edf97b61-1366-48f5-936f-5378dbccb550\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T11:35:07.81865953Z\",\n      \"endTime\": \"2026-09-30T11:54:13.175354213Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-x5s4hpmkwzhp-candidate-wsv85m--managed/providers/Microsoft.Compute/virtualMachines/v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-x5s4hpmkwzhp-candidate-wsv85m--managed/providers/Microsoft.Compute/virtualMachines/v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wsv85m\" for cluster \"cluster-candidate-4-23-wsv85m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7d6c0\u003e: \n    failed to create NodePool np-wsv85m: failed waiting for nodepool=\"np-wsv85m\" for cluster \"cluster-candidate-4-23-wsv85m\" in resourcegroup=\"rg-candidate-4-23-x5s4hpmkwzhp\" to finish creating: GET https://rp.j1071872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/edf97b61-1366-48f5-936f-5378dbccb550\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/edf97b61-1366-48f5-936f-5378dbccb550\",\n      \"name\": \"edf97b61-1366-48f5-936f-5378dbccb550\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T11:35:07.81865953Z\",\n      \"endTime\": \"2026-09-30T11:54:13.175354213Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-x5s4hpmkwzhp-candidate-wsv85m--managed/providers/Microsoft.Compute/virtualMachines/v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-x5s4hpmkwzhp-candidate-wsv85m--managed/providers/Microsoft.Compute/virtualMachines/v9b2q3h5h4o9b5j-np-wsv85m-qtz79-w22nd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c891abd031033d3cb3e9b62d8c25ec3e0489e46e07e6c599a1eb7e119494e08f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105244373301071872","occurred_at":"2026-09-30T10:32:26Z","failed_at":"alert","signature_id":"9e1c8df2837ce311616087aab77d355a705bb4d7065306ed80ed4c64b015fde4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T11:34:50Z\n  Ended:       2026-09-30T11:49:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1071872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5a636m6sfjl130rh16efne40festfg-cilium-cluster\", pod=\"router-7f64d88865-r8tm4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5a636m6sfjl130rh16efne40festfg-cilium-cluster/router-7f64d88865-r8tm4 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T11:35:51Z\n  Ended:       2026-09-30T11:49:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1071872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t5a636to94rd3t38eir1d61k2r3hvq2-m4p7l3e8z9w1p5p\", pod=\"router-5cc8888886-kjp9p\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t5a636to94rd3t38eir1d61k2r3hvq2-m4p7l3e8z9w1p5p/router-5cc8888886-kjp9p has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7262/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105242313000226816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7262,"pr_state":"closed","pr_sha":"4786601accbd2e57c0fa7b77dbd33b56d8d914be","final_merged_sha":"4786601accbd2e57c0fa7b77dbd33b56d8d914be","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T10:24:15Z","started_at":"2026-09-30T10:24:15Z","completed_at":"2026-09-30T12:48:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:54:00.226836742Z","timing_metadata_checked_at":"2026-09-30T12:54:00.226836742Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:54:00.229665233Z","region_metadata_checked_at":"2026-09-30T12:54:00.229665233Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105230891239346176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"closed","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"d98ef6349a3cd6d57591590dbb9cbff37ac89433","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:38:51Z","started_at":"2026-09-30T09:38:51Z","completed_at":"2026-09-30T12:25:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:32:29.943677602Z","timing_metadata_checked_at":"2026-09-30T12:32:29.943677602Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:32:29.951732175Z","region_metadata_checked_at":"2026-09-30T12:32:29.951732175Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b569b8cfc428bfe4c3ff82ac7bacc196b7d088712bcd514e00748d8c2ab7daab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105230891239346176","occurred_at":"2026-09-30T09:38:51Z","failed_at":"alert","signature_id":"6ce75aaaaf15744f114f30367d6df7d93358cb13c8e5de12639712815ed34e1d","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T12:05:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j9346176-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.63:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9346176/providers/microsoft.monitor/accounts/services-j9346176\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c94b5c67c-s5vnr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j9346176-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T12:05:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j9346176-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.63:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9346176/providers/microsoft.monitor/accounts/services-j9346176\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c94b5c67c-s5vnr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j9346176-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105230122981265408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"closed","pr_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","final_merged_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T09:35:49Z","started_at":"2026-09-30T09:35:49Z","completed_at":"2026-09-30T12:01:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:11:38.009358442Z","timing_metadata_checked_at":"2026-09-30T12:11:38.009358442Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:11:38.01243693Z","region_metadata_checked_at":"2026-09-30T12:11:38.01243693Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105229330492690432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6662,"pr_state":"open","pr_sha":"911c458b49d2a2cd3ced7ef2fefe8affa0c9e753","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:32:40Z","started_at":"2026-09-30T09:32:40Z","completed_at":"2026-09-30T11:45:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:50:20.5164356Z","timing_metadata_checked_at":"2026-09-30T11:50:20.5164356Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:50:20.519141491Z","region_metadata_checked_at":"2026-09-30T11:50:20.519141491Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"566a210ed34cfc77f20ea77c2401557688fd9e23870a67a02b9acc4cec81013f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105229330492690432","occurred_at":"2026-09-30T09:32:40Z","failed_at":"e2e","signature_id":"0c99b72686df644d7a91845938f6db64d74dcf85418abd2f5d73c4d6278f406f","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bd4f00\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-l57gw6mdwbj6\" to finish creating: GET https://rp.j2690432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75db90be-5602-433c-8319-190a1fa900d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75db90be-5602-433c-8319-190a1fa900d9\",\n      \"name\": \"75db90be-5602-433c-8319-190a1fa900d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T10:22:37.553403502Z\",\n      \"endTime\": \"2026-09-30T10:41:41.061493076Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-l57gw6mdwbj6--managed/providers/Microsoft.Compute/virtualMachines/d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-l57gw6mdwbj6--managed/providers/Microsoft.Compute/virtualMachines/d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bd4f00\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-l57gw6mdwbj6\" to finish creating: GET https://rp.j2690432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75db90be-5602-433c-8319-190a1fa900d9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75db90be-5602-433c-8319-190a1fa900d9\",\n      \"name\": \"75db90be-5602-433c-8319-190a1fa900d9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T10:22:37.553403502Z\",\n      \"endTime\": \"2026-09-30T10:41:41.061493076Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-l57gw6mdwbj6--managed/providers/Microsoft.Compute/virtualMachines/d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-l57gw6mdwbj6--managed/providers/Microsoft.Compute/virtualMachines/d1u4m0g7s7a4h7b-nodepool-128gib-pj9gv-krcrc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"closed","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"273677ce69aa7a8dc7e1da98b8942d3ea48b24d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:27:02Z","started_at":"2026-09-30T09:27:02Z","completed_at":"2026-09-30T10:38:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:56:56.826576217Z","timing_metadata_checked_at":"2026-09-30T10:56:56.826576217Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:56:56.829552107Z","region_metadata_checked_at":"2026-09-30T10:56:56.829552107Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"08a39272e9d55a927edf37292e4efdb7743e386b13006c794b70d3c46611f830","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","occurred_at":"2026-09-30T09:27:02Z","failed_at":"provision","signature_id":"d20ad03882fd0f5debbe72a144cd887757a3333ce3a6250ab7b88059925a6396","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T10:02:16.720Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T10:02:17.023Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j5090432-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j5090432-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T10:02:17.023Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:02:17.180Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T10:02:17.354Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T10:02:17.354Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:02:17.402Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:12:19.938Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T10:12:19.938Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T10:13:19.944Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T10:13:20.698Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j5090432-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j5090432-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T10:13:20.699Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:21.000Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T10:13:21.389Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:21.389Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:21.559Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T10:13:22.194Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:22.194Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:22.766Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:23.138Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T10:13:23.614Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T10:13:24.021Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:24.220Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:24.545Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:13:24.923Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T10:13:24.923Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:13:25.004Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-30T10:23:26.540Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T10:23:26.540Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-09-30T10:24:26.541Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-30T10:24:26.978Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j5090432-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:eb3e2e068172121a9ca133c5d17784df770dacc104f20c7d4a7df5f1c70e8a08 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:450m memory:472Mi]]] geoShortId:ca kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:canadacentral role:hcp-underlay-ci01-j5090432-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:5c29681bb4b38d08ca21c31c4553bab72e8e9c260dc0ec875e1d014268a9e580 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 region:canadacentral secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#e5a2e4f0-d338-424f-81b6-3ea36e90ffa8 INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-30T10:24:26.978Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:27.205Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-30T10:24:27.597Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:27.597Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:27.807Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T10:24:28.587Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:28.587Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:28.952Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:29.210Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T10:24:29.577Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-30T10:24:29.759Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:30.112Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:30.448Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-30T10:24:30.762Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-30T10:24:30.762Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-30T10:24:30.844Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-30T10:32:16.654Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-30T10:32:16.654Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"53f100d34948cd9dc6986263ea2fd212ca3e655ea8fc80f127573cd11309a2b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","occurred_at":"2026-09-30T09:27:02Z","failed_at":"alert","signature_id":"69c10044ecef2b03f895153e90c169d4ea0567b1642ce13c7454d66e14ce62ee","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:19:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j5090432-svc\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j5090432-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"b19de7a1cbb4b48460128795333acfd308de5a4e6ae9b5d5af73da0c894abadf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","occurred_at":"2026-09-30T09:27:02Z","failed_at":"alert","signature_id":"4ef2e890beff7c4f86db4449f89189712a00b70740bd3d6bf71d4bd36e6eada0","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:08:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-c997l\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-c997l has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:08:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-qptj8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-qptj8 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:08:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-69sbs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-69sbs has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"b4163a83cb2d9a4d62d63b769042e0c38a2c05be0bb8b00e8f0941330352b7f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227914915090432","occurred_at":"2026-09-30T09:27:02Z","failed_at":"alert","signature_id":"070a999d413963099d095402bcad5d0495e08e705056ebb3a3163bfbe8e380fa","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:18:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.69:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5090432/providers/microsoft.monitor/accounts/services-j5090432\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-gmtlp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:18:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j5090432-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.69:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5090432/providers/microsoft.monitor/accounts/services-j5090432\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-gmtlp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6919/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227543996010496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6919,"pr_state":"closed","pr_sha":"f5f7ba3708403bd680936e8b420e0d61595cc458","final_merged_sha":"f5f7ba3708403bd680936e8b420e0d61595cc458","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T09:25:34Z","started_at":"2026-09-30T09:25:34Z","completed_at":"2026-09-30T11:58:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:51.780522186Z","timing_metadata_checked_at":"2026-09-30T12:00:51.780522186Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:51.782843178Z","region_metadata_checked_at":"2026-09-30T12:00:51.782843178Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227448479125504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T09:25:11Z","started_at":"2026-09-30T09:25:11Z","completed_at":"2026-09-30T11:56:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:51.925279024Z","timing_metadata_checked_at":"2026-09-30T12:00:51.925279024Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:51.927668216Z","region_metadata_checked_at":"2026-09-30T12:00:51.927668216Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227431290867712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7220,"pr_state":"open","pr_sha":"540856c435b752648f8f78fe4a157f67de6eed0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:25:07Z","started_at":"2026-09-30T09:25:07Z","completed_at":"2026-09-30T11:47:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:50:21.052669628Z","timing_metadata_checked_at":"2026-09-30T11:50:21.052669628Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:50:21.059459806Z","region_metadata_checked_at":"2026-09-30T11:50:21.059459806Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4c32f3295b9628c046c828dd1281cb19208d6b85b2f7a37ba55f404911daa4a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227431290867712","occurred_at":"2026-09-30T09:25:07Z","failed_at":"e2e","signature_id":"f705bab8d4be596b76259e82bbce6d86611afa3edaf3393759077548f5abd2e5","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 611.396s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00158e610\u003e: \n    run command \"e2e-runcommand-1790764865640292876\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 611.396s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00158e610\u003e: \n    run command \"e2e-runcommand-1790764865640292876\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb120c9514f41ade3798c0918bf6c7e33712fa0ac08db0b23b0950e854a6cf03","failure_pattern":"run command \"e2e-runcommand-1790764865640292876\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1790764865640292876\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bb120c9514f41ade3798c0918bf6c7e33712fa0ac08db0b23b0950e854a6cf03"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227392430641152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7071,"pr_state":"open","pr_sha":"6e66d4c793650a413acda3f74dda6a03700e788a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:24:58Z","started_at":"2026-09-30T09:24:58Z","completed_at":"2026-09-30T10:22:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:35:01.316906616Z","timing_metadata_checked_at":"2026-09-30T10:35:01.316906616Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:35:01.319858106Z","region_metadata_checked_at":"2026-09-30T10:35:01.319858106Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"eda3d156511d7212744adbc99bfb6339be0dc2d6e6823a0fb19889471bf6bad6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105227392430641152","occurred_at":"2026-09-30T09:24:58Z","failed_at":"provision","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T10:09:47.479Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-09-30T10:09:48.123Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4]\"\ntime=2026-09-30T10:09:48.123Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-30T10:09:48.426Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-09-30T10:09:48.816Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-09-30T10:09:48.816Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-30T10:09:48.942Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-09-30T10:14:53.371Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"\ntime=2026-09-30T10:14:53.371Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abcd5f9b62e8cf7d13105a0c9ded33446cbd1e1f876e962101b7bbf9a2c604c6","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: InProgress, message: Job in progress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abcd5f9b62e8cf7d13105a0c9ded33446cbd1e1f876e962101b7bbf9a2c604c6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105226917077585920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"closed","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"c11d1aab795ebbd762ddef1d8188a105d875d002","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:23:03Z","started_at":"2026-09-30T09:23:03Z","completed_at":"2026-09-30T11:45:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:50:20.756488307Z","timing_metadata_checked_at":"2026-09-30T11:50:20.756488307Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:50:20.759215798Z","region_metadata_checked_at":"2026-09-30T11:50:20.759215798Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4de2c49c7698f94e5615fcb09cdebf543d09150287ec5f69dcad0ae2dccec1ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105226917077585920","occurred_at":"2026-09-30T09:23:03Z","failed_at":"e2e","signature_id":"4cbd0b39303c5ba882664f24927fa89f1b1e0d8bda43db16774956c03f9a6c68","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-7jbkqj\" with version 4.21.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011488e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-7jbkqj: failed waiting for cluster=\"cluster-zstream-4-21-7jbkqj\" in resourcegroup=\"rg-zstream-upgrade-4-21-7qqcshqrm588\" to finish creating: GET https://rp.j7585920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-7jbkqj\" with version 4.21.34 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011488e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-7jbkqj: failed waiting for cluster=\"cluster-zstream-4-21-7jbkqj\" in resourcegroup=\"rg-zstream-upgrade-4-21-7qqcshqrm588\" to finish creating: GET https://rp.j7585920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/30ab22cf-79b3-49a2-b5a2-b420840f6f78\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7176/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105226057555644416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7176,"pr_state":"closed","pr_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","final_merged_sha":"a90ad015f6e70d63790d33a15a5294c05dbf614b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T09:19:39Z","started_at":"2026-09-30T09:19:39Z","completed_at":"2026-09-30T11:45:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:50:20.918556771Z","timing_metadata_checked_at":"2026-09-30T11:50:20.918556771Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:50:20.92190686Z","region_metadata_checked_at":"2026-09-30T11:50:20.92190686Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105225624229515264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T09:17:55Z","started_at":"2026-09-30T09:17:55Z","completed_at":"2026-09-30T11:53:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:52.067749269Z","timing_metadata_checked_at":"2026-09-30T12:00:52.067749269Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:52.070315161Z","region_metadata_checked_at":"2026-09-30T12:00:52.070315161Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105225488111767552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7229,"pr_state":"closed","pr_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","final_merged_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T09:17:24Z","started_at":"2026-09-30T09:17:24Z","completed_at":"2026-09-30T11:58:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:52.210386913Z","timing_metadata_checked_at":"2026-09-30T12:00:52.210386913Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:52.215980295Z","region_metadata_checked_at":"2026-09-30T12:00:52.215980295Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105225367915597824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:16:55Z","started_at":"2026-09-30T09:16:55Z","completed_at":"2026-09-30T11:51:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T12:00:51.61643531Z","timing_metadata_checked_at":"2026-09-30T12:00:51.61643531Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T12:00:51.619536Z","region_metadata_checked_at":"2026-09-30T12:00:51.619536Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"81d160828454cbc25ad9e4d32319090ce9770a5d36b8678570dbd5dee05cee99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105225367915597824","occurred_at":"2026-09-30T09:16:55Z","failed_at":"e2e","signature_id":"0004ec9246093dd32ccef6900517fe1c26f01933d048bec9e64d50cce9f0f872","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xkg7gf\" for cluster \"cluster-candidate-4-20-xkg7gf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001435b60\u003e: \n    failed to create NodePool np-xkg7gf: failed waiting for nodepool=\"np-xkg7gf\" for cluster \"cluster-candidate-4-20-xkg7gf\" in resourcegroup=\"rg-candidate-4-20-j47p9rbxx46g\" to finish creating: GET https://rp.j5597824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26468792-a524-4bf9-970b-d4edc2f9bfe8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26468792-a524-4bf9-970b-d4edc2f9bfe8\",\n      \"name\": \"26468792-a524-4bf9-970b-d4edc2f9bfe8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T10:25:29.553885983Z\",\n      \"endTime\": \"2026-09-30T10:44:30.082076643Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-j47p9rbxx46g-candidate-xkg7gf--managed/providers/Microsoft.Compute/virtualMachines/q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-j47p9rbxx46g-candidate-xkg7gf--managed/providers/Microsoft.Compute/virtualMachines/q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xkg7gf\" for cluster \"cluster-candidate-4-20-xkg7gf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001435b60\u003e: \n    failed to create NodePool np-xkg7gf: failed waiting for nodepool=\"np-xkg7gf\" for cluster \"cluster-candidate-4-20-xkg7gf\" in resourcegroup=\"rg-candidate-4-20-j47p9rbxx46g\" to finish creating: GET https://rp.j5597824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26468792-a524-4bf9-970b-d4edc2f9bfe8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26468792-a524-4bf9-970b-d4edc2f9bfe8\",\n      \"name\": \"26468792-a524-4bf9-970b-d4edc2f9bfe8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T10:25:29.553885983Z\",\n      \"endTime\": \"2026-09-30T10:44:30.082076643Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-j47p9rbxx46g-candidate-xkg7gf--managed/providers/Microsoft.Compute/virtualMachines/q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-j47p9rbxx46g-candidate-xkg7gf--managed/providers/Microsoft.Compute/virtualMachines/q2o4d5p5y8u3o9e-np-xkg7gf-jxzwr-xwjrg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105224754142121984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6944,"pr_state":"open","pr_sha":"d00b03862945426285f54b76cb1f24ccfc8fdbcb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T09:14:29Z","started_at":"2026-09-30T09:14:29Z","completed_at":"2026-09-30T10:12:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:24:49.799005238Z","timing_metadata_checked_at":"2026-09-30T10:24:49.799005238Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:24:49.80462712Z","region_metadata_checked_at":"2026-09-30T10:24:49.80462712Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"473175968bc09b55161db8fa2d0daffee39ba4bbbe5274eae9a86569cafb3a19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105224754142121984","occurred_at":"2026-09-30T09:14:29Z","failed_at":"provision","signature_id":"72c70d03904798e23a81b751f90dae41f83d98201724acd02612ebc1b536daae","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-09-30T10:01:25.496Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-30T10:01:25.890Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperPoolCert-34 clientId:72671881-beb9-4028-bc7c-47f511281566 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpci01oidcj2121984.z9.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj2121984.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] backupCadenceProfile:testing backupScheduleState:Disabled clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:072ccd06-457a-49cc-a612-ad4dbbc199a3 databaseName:arohcpci01-rp-j2121984 databaseUrl:https://arohcpci01-rp-j2121984.documents.azure.com:443/ kubeApplierCosmosContainerName:Manifests-MC-1 location:canadacentral] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-gq2sycdx/pipeline@sha256:ec35d9956c06798bf0f6c7e4116d8b1c120ff9a5a7cbe4e991d36cae1bea464e replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:700m memory:1Gi]] zoneCount:3] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] logVerbosity:4 maestro:map[sourceEnvironmentIdentifier:arohcpci01] miMock:map[certName:msiMockPoolCert-18 clientId:8c088886-a118-4973-8168-0d7ae1493aa7 clientIdSecretName: principalId:ccbf438f-08ec-4e2b-ae4e-219ce7dc3762 principalIdSecretName: tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-gq2sycdx/pipeline:pull workloadIdentityClientId:7cbf62ec-0a3a-4039-a0c0-7c88aa6f75bf workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:072ccd06-457a-49cc-a612-ad4dbbc199a3 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv]]\"\ntime=2026-09-30T10:01:25.890Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-30T10:01:26.074Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-09-30T10:01:26.234Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-09-30T10:01:26.234Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-30T10:01:26.298Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-09-30T10:06:31.687Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded\"\ntime=2026-09-30T10:06:31.687Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e8bcec5029342ecf0af7106c015ff83add435f419b54399997693a996c71892f","failure_pattern":"failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 0/2 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/aro-hcp/aro-hcp-backend not ready. status: InProgress, message: Available: 0/2\\ncontext deadline exceeded"}},{"row_id":"75fb01d4f64918af81a1a83d534a6c01bfe962c6974dec4710ba836a01c9ad01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105224754142121984","occurred_at":"2026-09-30T09:14:29Z","failed_at":"alert","signature_id":"960817798cecef5f242862956f5ea60bc993965ed8ff7b4209f48c2ae021c775","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 95 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-rsvtx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-rsvtx has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-796x2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-796x2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-pcjcl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-pcjcl has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-z998l\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-z998l has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-m8698\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-m8698 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-82cr4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-82cr4 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-pzjb4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-pzjb4 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-7x96q\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-7x96q has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-x2wph\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-x2wph has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-n9dj7\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-n9dj7 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-nwhcl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-nwhcl has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-vgjxq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-vgjxq has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-5nrdf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-5nrdf has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-m52wd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-m52wd has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-5g7dm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-5g7dm has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-sqlsb\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-sqlsb has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-vfwjz\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-vfwjz has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-czw9d\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-czw9d has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-ft9sh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-ft9sh has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-2lclr\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-2lclr has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-tgq28\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-tgq28 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-qvg8l\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-qvg8l has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-js2ms\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-js2ms has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-s2mtg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-s2mtg has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-g76kv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-g76kv has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-6rcxw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-6rcxw has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-sw7b9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-sw7b9 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-c4n6b\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-c4n6b has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-lq6ck\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-lq6ck has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-dvlrh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-dvlrh has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-cj9md\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-cj9md has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-bjcl4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-bjcl4 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-48gt2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-48gt2 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-qchw5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-qchw5 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-bz68s\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-bz68s has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-tv8hg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-tv8hg has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-8n25z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-8n25z has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-n4r6z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-n4r6z has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-gpj9m\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-gpj9m has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-f4z6c\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-f4z6c has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-cd9g2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-cd9g2 has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-vqj8z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-vqj8z has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-zm597\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-zm597 has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-q6m5f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-q6m5f has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-bxhlp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-bxhlp has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-8c6pj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-8c6pj has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-gjs74\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-gjs74 has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-lrcrl\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-lrcrl has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-kwfr5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-kwfr5 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-jnhrj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-jnhrj has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-j2x54\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-j2x54 has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-m4w5z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-m4w5z has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-c5jzp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-c5jzp has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-xtl8c\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-xtl8c has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-pk965\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-pk965 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-xc2p8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-xc2p8 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-jkhgh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-jkhgh has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-vl6w8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-vl6w8 has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-9xhbm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-9xhbm has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-pldcj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-pldcj has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-wzvnf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-wzvnf has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-c8pn2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-c8pn2 has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-7dgv9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-7dgv9 has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-cw6k5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-cw6k5 has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-cp898\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-cp898 has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-vfv5j\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-vfv5j has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-mcbpb\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-mcbpb has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-fq9qk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-fq9qk has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-v2www\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-v2www has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-8j9r8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-8j9r8 has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-dtv87\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-dtv87 has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-pk6pc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-pk6pc has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-jpfz4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-jpfz4 has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-v6ml2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-v6ml2 has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-msxpq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-msxpq has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-hbpsf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-hbpsf has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-5njm9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-5njm9 has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-2nc5h\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-2nc5h has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-7ftmt\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-7ftmt has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-ccxrv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-ccxrv has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-ngg9z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-ngg9z has been in a non-ready state for longer than 5 minutes.\nFiring 82:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-8w6bk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-8w6bk has been in a non-ready state for longer than 5 minutes.\nFiring 83:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-jgzjj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-jgzjj has been in a non-ready state for longer than 5 minutes.\nFiring 84:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-4dchf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-4dchf has been in a non-ready state for longer than 5 minutes.\nFiring 85:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-dkq6z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-dkq6z has been in a non-ready state for longer than 5 minutes.\nFiring 86:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-vvrhd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-vvrhd has been in a non-ready state for longer than 5 minutes.\nFiring 87:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-fllz9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-fllz9 has been in a non-ready state for longer than 5 minutes.\nFiring 88:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-jckvz\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-jckvz has been in a non-ready state for longer than 5 minutes.\nFiring 89:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-2fn44\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-2fn44 has been in a non-ready state for longer than 5 minutes.\nFiring 90:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-k5psj\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-k5psj has been in a non-ready state for longer than 5 minutes.\nFiring 91:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-4r64q\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-4r64q has been in a non-ready state for longer than 5 minutes.\nFiring 92:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-98h9z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-98h9z has been in a non-ready state for longer than 5 minutes.\nFiring 93:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-9b6sv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-9b6sv has been in a non-ready state for longer than 5 minutes.\nFiring 94:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-sfw6f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-sfw6f has been in a non-ready state for longer than 5 minutes.\nFiring 95:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T10:05:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j2121984-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-6285h\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod prometheus/prometheus-operator-5d45794ff-6285h has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","e8bcec5029342ecf0af7106c015ff83add435f419b54399997693a996c71892f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105221039314178048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7221,"pr_state":"closed","pr_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","final_merged_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T08:59:43Z","started_at":"2026-09-30T08:59:43Z","completed_at":"2026-09-30T11:24:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:39:37.613479559Z","timing_metadata_checked_at":"2026-09-30T11:39:37.613479559Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:39:37.618252442Z","region_metadata_checked_at":"2026-09-30T11:39:37.618252442Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105219090921558016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7104,"pr_state":"closed","pr_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","final_merged_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T08:51:58Z","started_at":"2026-09-30T08:51:58Z","completed_at":"2026-09-30T11:14:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T11:29:06.920150004Z","timing_metadata_checked_at":"2026-09-30T11:29:06.920150004Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T11:29:06.924542991Z","region_metadata_checked_at":"2026-09-30T11:29:06.924542991Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7262/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105203604242567168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7262,"pr_state":"closed","pr_sha":"4786601accbd2e57c0fa7b77dbd33b56d8d914be","final_merged_sha":"4786601accbd2e57c0fa7b77dbd33b56d8d914be","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T07:50:26Z","started_at":"2026-09-30T07:50:26Z","completed_at":"2026-09-30T10:13:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:24:49.951983251Z","timing_metadata_checked_at":"2026-09-30T10:24:49.951983251Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:24:49.95531914Z","region_metadata_checked_at":"2026-09-30T10:24:49.95531914Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"71a0a4c4672902b2859d1d4f3ac27c03ce51222006fc99ec96d050216667aa0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7262/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105203604242567168","occurred_at":"2026-09-30T07:50:26Z","failed_at":"e2e","signature_id":"7c21d39ce60d2fc86f8e44cd27976742c9868e16e557b49cd66e0e9f7da03ea6","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002511248\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xk42m82bkxtn\" to finish creating: GET https://rp.j2567168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/034a0762-ab5c-48da-87dd-8d208861a970\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/034a0762-ab5c-48da-87dd-8d208861a970\",\n      \"name\": \"034a0762-ab5c-48da-87dd-8d208861a970\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:55:45.833258741Z\",\n      \"endTime\": \"2026-09-30T09:14:52.939311971Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-nssc6-7nz65: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-nssc6-7nz65: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-nssc6-7nz65\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-nssc6-7nz65\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-nssc6-7nz65' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002511248\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xk42m82bkxtn\" to finish creating: GET https://rp.j2567168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/034a0762-ab5c-48da-87dd-8d208861a970\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/034a0762-ab5c-48da-87dd-8d208861a970\",\n      \"name\": \"034a0762-ab5c-48da-87dd-8d208861a970\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:55:45.833258741Z\",\n      \"endTime\": \"2026-09-30T09:14:52.939311971Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-nssc6-7nz65: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-nssc6-7nz65: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    encountered errors downloading console logs:\n      - VM \"cilium-cluster-cilium-np-nssc6-7nz65\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-nssc6-7nz65\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-xk42m82bkxtn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-nssc6-7nz65/retrieveBootDiagnosticsData\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: OperationNotAllowed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"OperationNotAllowed\",\n        \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-nssc6-7nz65' since another operation is in progress.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105193266038181888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"closed","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"d98ef6349a3cd6d57591590dbb9cbff37ac89433","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T07:09:21Z","started_at":"2026-09-30T07:09:21Z","completed_at":"2026-09-30T09:29:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:42:58.821221896Z","timing_metadata_checked_at":"2026-09-30T09:42:58.821221896Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:42:58.824746685Z","region_metadata_checked_at":"2026-09-30T09:42:58.824746685Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"52b438b71436b28a7d3ad3bdf1d1b7bc36afe9b237858b94d05937154b109a4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105193266038181888","occurred_at":"2026-09-30T07:09:21Z","failed_at":"e2e","signature_id":"64488f0586b5477bcb601e48c11446bba99626aed3e38dfbb290b68d5bcdd29e","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aae2c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-nkgjwtvv229n\" to finish creating: GET https://rp.j8181888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\",\n      \"name\": \"4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:33:01.043737315Z\",\n      \"endTime\": \"2026-09-30T08:52:07.873742657Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-nkgjwtvv229n--managed/providers/Microsoft.Compute/virtualMachines/j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-nkgjwtvv229n--managed/providers/Microsoft.Compute/virtualMachines/j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aae2c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-nkgjwtvv229n\" to finish creating: GET https://rp.j8181888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\",\n      \"name\": \"4fc933c7-0d41-4651-a4a8-b2c8e1264ee6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:33:01.043737315Z\",\n      \"endTime\": \"2026-09-30T08:52:07.873742657Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-nkgjwtvv229n--managed/providers/Microsoft.Compute/virtualMachines/j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-nkgjwtvv229n--managed/providers/Microsoft.Compute/virtualMachines/j6f0a3z3m5a4v2o-np-1-cdxdq-nmfh9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105192683818455040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T07:06:56Z","started_at":"2026-09-30T07:06:56Z","completed_at":"2026-09-30T09:27:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:42:58.841921029Z","timing_metadata_checked_at":"2026-09-30T09:42:58.841921029Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:42:58.844741519Z","region_metadata_checked_at":"2026-09-30T09:42:58.844741519Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105191830118207488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7182,"pr_state":"closed","pr_sha":"b56a252408226d8ed3ec180421d2845783ace28c","final_merged_sha":"b56a252408226d8ed3ec180421d2845783ace28c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T07:03:39Z","started_at":"2026-09-30T07:03:39Z","completed_at":"2026-09-30T09:26:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:42:58.861493365Z","timing_metadata_checked_at":"2026-09-30T09:42:58.861493365Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:42:58.864362055Z","region_metadata_checked_at":"2026-09-30T09:42:58.864362055Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7260/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105189267666898944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7260,"pr_state":"closed","pr_sha":"d2063f84c5105758d4ff2370d77bd0c238c14b4d","final_merged_sha":"e2029a06a51da6cdf2ed5b5ef1c620dcc9cda152","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T06:53:28Z","started_at":"2026-09-30T06:53:28Z","completed_at":"2026-09-30T09:17:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:32:36.719167257Z","timing_metadata_checked_at":"2026-09-30T09:32:36.719167257Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:32:36.722742946Z","region_metadata_checked_at":"2026-09-30T09:32:36.722742946Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a98b5e849cc54afb43c09f0ac45ea0cf93e44f1c4ff1cfa3a1d7b10ff53b2af5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7260/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105189267666898944","occurred_at":"2026-09-30T06:53:28Z","failed_at":"e2e","signature_id":"87fbac54cf2eeb9a961209caab24269852fdc963c5e9568aa686fbe5fb493555","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c97540\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4vhtkz65lcdf\" to finish creating: GET https://rp.j6898944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f923a9cc-3bf2-46c5-a14f-9498c8c16cec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f923a9cc-3bf2-46c5-a14f-9498c8c16cec\",\n      \"name\": \"f923a9cc-3bf2-46c5-a14f-9498c8c16cec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:58:35.462676175Z\",\n      \"endTime\": \"2026-09-30T08:17:44.959895115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4vhtkz65lcdf--managed/providers/Microsoft.Compute/virtualMachines/v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4vhtkz65lcdf--managed/providers/Microsoft.Compute/virtualMachines/v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c97540\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4vhtkz65lcdf\" to finish creating: GET https://rp.j6898944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f923a9cc-3bf2-46c5-a14f-9498c8c16cec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f923a9cc-3bf2-46c5-a14f-9498c8c16cec\",\n      \"name\": \"f923a9cc-3bf2-46c5-a14f-9498c8c16cec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:58:35.462676175Z\",\n      \"endTime\": \"2026-09-30T08:17:44.959895115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4vhtkz65lcdf--managed/providers/Microsoft.Compute/virtualMachines/v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-4vhtkz65lcdf--managed/providers/Microsoft.Compute/virtualMachines/v6y0n4l6q1w8i3p-np-delete-main-p2bwr-dv274 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4t7wr4mxhmz6\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7261,"pr_state":"closed","pr_sha":"eb16c2e94e7f2596a478e5570733fb66ffcca02f","final_merged_sha":"f6667070546b139bd53a7df7d7674fd0c4250636","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T06:47:34Z","started_at":"2026-09-30T06:47:34Z","completed_at":"2026-09-30T10:35:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:45:58.664659857Z","timing_metadata_checked_at":"2026-09-30T10:45:58.664659857Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:45:58.667841546Z","region_metadata_checked_at":"2026-09-30T10:45:58.667841546Z"},"failed_at":["alert","e2e"],"failed_test_count":35,"occurrences":[{"row_id":"0db939d85773dffea3b3d8a6da5d5473aad64b08aa88cdb877774f3eb5d9d230","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"600a0314fbb714dba09f339df24ce54ab67395260e920baf7aff2d02983e4161","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014003d8\u003e: \n    resource group \"admin-credential-lifecycle-test-5rfj4vwnhrwj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-vxghsm\" in resourcegroup=\"admin-credential-lifecycle-test-5rfj4vwnhrwj\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0efef022-0122-4549-9e52-3393af9918c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0efef022-0122-4549-9e52-3393af9918c1\",\n      \"name\": \"0efef022-0122-4549-9e52-3393af9918c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:44:51.882854873Z\",\n      \"endTime\": \"2026-09-30T08:08:52.436086515Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofgpftr9snm78jc1alvisvlihdkq still exists (deletion dispatched at 2026-09-30T07:45:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014003d8\u003e: \n    resource group \"admin-credential-lifecycle-test-5rfj4vwnhrwj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-vxghsm\" in resourcegroup=\"admin-credential-lifecycle-test-5rfj4vwnhrwj\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0efef022-0122-4549-9e52-3393af9918c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0efef022-0122-4549-9e52-3393af9918c1\",\n      \"name\": \"0efef022-0122-4549-9e52-3393af9918c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:44:51.882854873Z\",\n      \"endTime\": \"2026-09-30T08:08:52.436086515Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofgpftr9snm78jc1alvisvlihdkq still exists (deletion dispatched at 2026-09-30T07:45:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1912ef3c9abb82e056d4056b8207722c9fd167a57f3f29e5a842e8be8f901bfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"e8ae7ca73bac34509929f98cbc2e291873367df09e30b7ee513a70980408c7e4","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0009f4378\u003e: \n    resource group \"v20261001preview-glc4g8rxxg96\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-glc4g8rxxg96\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22941031-f785-42ec-930f-02f9b4baf5df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22941031-f785-42ec-930f-02f9b4baf5df\",\n      \"name\": \"22941031-f785-42ec-930f-02f9b4baf5df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:50:24.203578528Z\",\n      \"endTime\": \"2026-09-30T08:14:32.552450638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obpih7ob1nmt9oif2arpkqr6ctja still exists (deletion dispatched at 2026-09-30T07:50:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0009f4378\u003e: \n    resource group \"v20261001preview-glc4g8rxxg96\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-glc4g8rxxg96\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22941031-f785-42ec-930f-02f9b4baf5df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22941031-f785-42ec-930f-02f9b4baf5df\",\n      \"name\": \"22941031-f785-42ec-930f-02f9b4baf5df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:50:24.203578528Z\",\n      \"endTime\": \"2026-09-30T08:14:32.552450638Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obpih7ob1nmt9oif2arpkqr6ctja still exists (deletion dispatched at 2026-09-30T07:50:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"19bd4c954029e6d57b43b641a1d2597db83c717994e1f1284f8e6a0ea5355ff1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"c425dbb349c3cb4e0d438e7efa64caf05e63c738219783e383f7f9abd3c127ca","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e06228\u003e: \n    resource group \"rg-np-version-upgrade-lhpwgz-8wlwbkbk8nl7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-lhpwgz\" in resourcegroup=\"rg-np-version-upgrade-lhpwgz-8wlwbkbk8nl7\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32343bad-f62c-41f1-a78a-34ece808734d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32343bad-f62c-41f1-a78a-34ece808734d\",\n      \"name\": \"32343bad-f62c-41f1-a78a-34ece808734d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:09:00.777952873Z\",\n      \"endTime\": \"2026-09-30T08:33:06.459889594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oj68ut6sfef5ef3g3kgp5llvpbpr still exists (deletion dispatched at 2026-09-30T08:09:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e06228\u003e: \n    resource group \"rg-np-version-upgrade-lhpwgz-8wlwbkbk8nl7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-lhpwgz\" in resourcegroup=\"rg-np-version-upgrade-lhpwgz-8wlwbkbk8nl7\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32343bad-f62c-41f1-a78a-34ece808734d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32343bad-f62c-41f1-a78a-34ece808734d\",\n      \"name\": \"32343bad-f62c-41f1-a78a-34ece808734d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:09:00.777952873Z\",\n      \"endTime\": \"2026-09-30T08:33:06.459889594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oj68ut6sfef5ef3g3kgp5llvpbpr still exists (deletion dispatched at 2026-09-30T08:09:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2119355726f4cf32f3822627de96caa5ededfa4274abf0e585a01a6b30ac492b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"a063820c19d1cfffec5676eb083b22917baff3755e10e5f7ba1e4ca7f468b87e","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c0e078\u003e: \n    resource group \"admin-api-breakglass-bvjprckvbd9j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-bvjprckvbd9j\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caa65f31-7834-40ae-9558-2b145d3d3707\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caa65f31-7834-40ae-9558-2b145d3d3707\",\n      \"name\": \"caa65f31-7834-40ae-9558-2b145d3d3707\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:42:39.486123591Z\",\n      \"endTime\": \"2026-09-30T08:06:42.572288271Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oguqit52ovjjoaakam9v60fven2c still exists (deletion dispatched at 2026-09-30T07:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c0e078\u003e: \n    resource group \"admin-api-breakglass-bvjprckvbd9j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-bvjprckvbd9j\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caa65f31-7834-40ae-9558-2b145d3d3707\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caa65f31-7834-40ae-9558-2b145d3d3707\",\n      \"name\": \"caa65f31-7834-40ae-9558-2b145d3d3707\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:42:39.486123591Z\",\n      \"endTime\": \"2026-09-30T08:06:42.572288271Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oguqit52ovjjoaakam9v60fven2c still exists (deletion dispatched at 2026-09-30T07:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"29b4cfd2569daa9c3d146129fdf2fc655434231fff8fc418d03d763d187358fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"4990898d8a8d8d7b3cccfd436135763eddfa8bf40bd6a2221cf0582f059cda21","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e84138\u003e: \n    resource group \"rg-zstream-upgrade-4-21-d6bd8lqnx9k6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-mpqc9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-d6bd8lqnx9k6\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9d0d023-161e-4fae-bc07-fa4aac05780f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9d0d023-161e-4fae-bc07-fa4aac05780f\",\n      \"name\": \"f9d0d023-161e-4fae-bc07-fa4aac05780f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:56:48.455291699Z\",\n      \"endTime\": \"2026-09-30T08:20:52.406066017Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofmkuvbuga9h9ge40bfgpgl5gvfp still exists (deletion dispatched at 2026-09-30T07:57:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e84138\u003e: \n    resource group \"rg-zstream-upgrade-4-21-d6bd8lqnx9k6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-mpqc9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-d6bd8lqnx9k6\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9d0d023-161e-4fae-bc07-fa4aac05780f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9d0d023-161e-4fae-bc07-fa4aac05780f\",\n      \"name\": \"f9d0d023-161e-4fae-bc07-fa4aac05780f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:56:48.455291699Z\",\n      \"endTime\": \"2026-09-30T08:20:52.406066017Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofmkuvbuga9h9ge40bfgpgl5gvfp still exists (deletion dispatched at 2026-09-30T07:57:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d105cebe60ed6812e94fadbef661524145d3c21ce52d8c0b49709a521e72963","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"afe091df3b6537610ba7d13c6ce02b998c9878ab1c5f2da66b17f37fa47b59c2","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00156e408\u003e: \n    resource group \"private-ingress-wkhlpqct8tlb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-wkhlpqct8tlb\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\",\n      \"name\": \"a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:47:33.546137807Z\",\n      \"endTime\": \"2026-09-30T08:11:42.306304509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oboi9api106r8emc4efcckh95di9 still exists (deletion dispatched at 2026-09-30T07:47:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00156e408\u003e: \n    resource group \"private-ingress-wkhlpqct8tlb\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-wkhlpqct8tlb\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\",\n      \"name\": \"a9cccb3e-9d83-49b8-9aa6-d3a585bd59e7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:47:33.546137807Z\",\n      \"endTime\": \"2026-09-30T08:11:42.306304509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oboi9api106r8emc4efcckh95di9 still exists (deletion dispatched at 2026-09-30T07:47:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"457c2f9baaabbe672ba588083f8d07313b8a27d17ef5c1f745fa7ee5a4ff568f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"61d2f9db9ee4e5c8023a43eed001242fe66442b4bd834571d5f4ea2180946254","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011481e0\u003e: \n    resource group \"rg-cluster-back-version-cnrv4422rs76\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011481e0\u003e: \n    resource group \"rg-cluster-back-version-cnrv4422rs76\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-cnrv4422rs76\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-cnrv4422rs76\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-cnrv4422rs76, error: context deadline exceeded"}},{"row_id":"4710b0b2c0bbbb32df126dd783e1defe83e010505e6607bf334dab28576a4aa1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"f7618045d7cfaab4bd66f07a97cf823014f39a37fe0a4c093e781b82a7306d83","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00095ea98\u003e: \n    resource group \"ephemeral-osdisk-zwnb5bhrdzqz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zwnb5bhrdzqz\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c5b0070-844b-4562-99f3-47e51a6c15ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c5b0070-844b-4562-99f3-47e51a6c15ef\",\n      \"name\": \"9c5b0070-844b-4562-99f3-47e51a6c15ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:45:01.77044017Z\",\n      \"endTime\": \"2026-09-30T08:09:02.54290012Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ok2d2baqjscldo4vmovim4i18pgh still exists (deletion dispatched at 2026-09-30T07:45:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00095ea98\u003e: \n    resource group \"ephemeral-osdisk-zwnb5bhrdzqz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zwnb5bhrdzqz\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c5b0070-844b-4562-99f3-47e51a6c15ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c5b0070-844b-4562-99f3-47e51a6c15ef\",\n      \"name\": \"9c5b0070-844b-4562-99f3-47e51a6c15ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:45:01.77044017Z\",\n      \"endTime\": \"2026-09-30T08:09:02.54290012Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ok2d2baqjscldo4vmovim4i18pgh still exists (deletion dispatched at 2026-09-30T07:45:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4df2049e8ef6569c78e3875a8eedf4e39b331a6a7d0ea8f9c08af8d47d2d492d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"417b6f53e945c7f505f4ed684b6bc1182432048a48254ebedbdc5118b22e4c97","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001344078\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-fs5gn7c5tqxf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-56tt5z\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-fs5gn7c5tqxf\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\",\n      \"name\": \"52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:59:56.020744554Z\",\n      \"endTime\": \"2026-09-30T08:24:02.403347508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odktt8bp0otlfbosu69ogttm3nlc still exists (deletion dispatched at 2026-09-30T07:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001344078\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-fs5gn7c5tqxf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-56tt5z\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-fs5gn7c5tqxf\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\",\n      \"name\": \"52d0c5fd-02ac-47eb-a5f2-62e5ca81cff9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:59:56.020744554Z\",\n      \"endTime\": \"2026-09-30T08:24:02.403347508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odktt8bp0otlfbosu69ogttm3nlc still exists (deletion dispatched at 2026-09-30T07:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"595ec19c6f6b5ff6a4579877bdc7fa2530aa28fa16fb6bf7e6e3937eae6902cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"f4000e677b431fd645fc9061bdd557db73077acbe3281573024aff7d3154db6e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:55:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0283264-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-84tpd\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-84tpd (namespace arobit) on cluster ci01-j0283264-mgmt-2 has averaged 123.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"5a0953dc593a812609375791f8764824206d037c2bcae1ae50a10f82321d99e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"ffa29da8af83e7964daeb9af651866d04306dbb88818e3e1019cbb1d504218e6","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00106a9d8\u003e: \n    resource group \"pullsecret-test-7dnj5wrv5xms\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7dnj5wrv5xms\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c782daed-b306-467e-81e3-8b6dd06ad52a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c782daed-b306-467e-81e3-8b6dd06ad52a\",\n      \"name\": \"c782daed-b306-467e-81e3-8b6dd06ad52a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:52.027158436Z\",\n      \"endTime\": \"2026-09-30T08:13:52.469740968Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofor9qrgni6pa0caelkcpsahuun0 still exists (deletion dispatched at 2026-09-30T07:49:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00106a9d8\u003e: \n    resource group \"pullsecret-test-7dnj5wrv5xms\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7dnj5wrv5xms\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c782daed-b306-467e-81e3-8b6dd06ad52a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c782daed-b306-467e-81e3-8b6dd06ad52a\",\n      \"name\": \"c782daed-b306-467e-81e3-8b6dd06ad52a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:52.027158436Z\",\n      \"endTime\": \"2026-09-30T08:13:52.469740968Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofor9qrgni6pa0caelkcpsahuun0 still exists (deletion dispatched at 2026-09-30T07:49:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5f3a2da81128ee3d309a1a135e4ec780d489500546e49753ac7e25044751d3b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"c635c7f3ad43e5d41b74812d30f60963314806e81ba8a3813c7a319c4b22a212","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b6e0a8\u003e: \n    resource group \"customer-rg-hd4bhsrz7d7n\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hd4bhsrz7d7n\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b56195ce-49cf-4cb1-b7c9-571ff13499d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b56195ce-49cf-4cb1-b7c9-571ff13499d0\",\n      \"name\": \"b56195ce-49cf-4cb1-b7c9-571ff13499d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:41:31.959909446Z\",\n      \"endTime\": \"2026-09-30T08:05:32.486384816Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56onftd04005nl721veuiicq8a5s6u still exists (deletion dispatched at 2026-09-30T07:41:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001b6e0a8\u003e: \n    resource group \"customer-rg-hd4bhsrz7d7n\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hd4bhsrz7d7n\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b56195ce-49cf-4cb1-b7c9-571ff13499d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b56195ce-49cf-4cb1-b7c9-571ff13499d0\",\n      \"name\": \"b56195ce-49cf-4cb1-b7c9-571ff13499d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:41:31.959909446Z\",\n      \"endTime\": \"2026-09-30T08:05:32.486384816Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56onftd04005nl721veuiicq8a5s6u still exists (deletion dispatched at 2026-09-30T07:41:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"636a032946e462d2a6e4384a327344addbea1c3f9e8e70631735f2d029efe071","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"1922b6404a193a0248ac9de20c3029fc58e5a940536058fc0aa8443f32082444","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001baa138\u003e: \n    resource group \"np-autoscaling-x6vzkczm54sv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-x6vzkczm54sv\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be95d509-ef21-4815-a213-b7566866b60c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be95d509-ef21-4815-a213-b7566866b60c\",\n      \"name\": \"be95d509-ef21-4815-a213-b7566866b60c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:57:15.728931938Z\",\n      \"endTime\": \"2026-09-30T08:21:22.412329675Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oig3m7l5t041s2h3535h24baigrd still exists (deletion dispatched at 2026-09-30T07:57:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001baa138\u003e: \n    resource group \"np-autoscaling-x6vzkczm54sv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-x6vzkczm54sv\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be95d509-ef21-4815-a213-b7566866b60c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be95d509-ef21-4815-a213-b7566866b60c\",\n      \"name\": \"be95d509-ef21-4815-a213-b7566866b60c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:57:15.728931938Z\",\n      \"endTime\": \"2026-09-30T08:21:22.412329675Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oig3m7l5t041s2h3535h24baigrd still exists (deletion dispatched at 2026-09-30T07:57:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"65690483a5d0925dcd7b439367e4c9415af693973b4f0d523e3872a97a976866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"9825bceafad7b29cf6ff082816b91e1a07777df23ef630f76b47894fe3017f42","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002180138\u003e: \n    resource group \"feature-aggregation-7452j745q24m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7452j745q24m\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a344fed7-604f-4718-a10f-cca3aa460a69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a344fed7-604f-4718-a10f-cca3aa460a69\",\n      \"name\": \"a344fed7-604f-4718-a10f-cca3aa460a69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:03:38.750208141Z\",\n      \"endTime\": \"2026-09-30T08:27:42.421715531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56pklkbe3f9k9tk3gemfqinnl5654u still exists (deletion dispatched at 2026-09-30T08:03:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc002180138\u003e: \n    resource group \"feature-aggregation-7452j745q24m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7452j745q24m\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a344fed7-604f-4718-a10f-cca3aa460a69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a344fed7-604f-4718-a10f-cca3aa460a69\",\n      \"name\": \"a344fed7-604f-4718-a10f-cca3aa460a69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:03:38.750208141Z\",\n      \"endTime\": \"2026-09-30T08:27:42.421715531Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56pklkbe3f9k9tk3gemfqinnl5654u still exists (deletion dispatched at 2026-09-30T08:03:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"711603b0733fa1910389f77e88ed94da115ca485187912d5966d895609fa454a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"397097b128fb1f71d9cac290dbcc3acd3e5a33bf08922e38de2034ce0a98cded","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0018c61c8\u003e: \n    resource group \"cni-cilium-5mwkmtbx9lm6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5mwkmtbx9lm6\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc69ed49-5ecc-40ea-9a0d-84c38d022d57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc69ed49-5ecc-40ea-9a0d-84c38d022d57\",\n      \"name\": \"fc69ed49-5ecc-40ea-9a0d-84c38d022d57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:47:13.892958657Z\",\n      \"endTime\": \"2026-09-30T08:11:22.315778755Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojjtudpbnkdj9hn5lbovlb6qujk0 still exists (deletion dispatched at 2026-09-30T07:47:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0018c61c8\u003e: \n    resource group \"cni-cilium-5mwkmtbx9lm6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5mwkmtbx9lm6\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc69ed49-5ecc-40ea-9a0d-84c38d022d57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc69ed49-5ecc-40ea-9a0d-84c38d022d57\",\n      \"name\": \"fc69ed49-5ecc-40ea-9a0d-84c38d022d57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:47:13.892958657Z\",\n      \"endTime\": \"2026-09-30T08:11:22.315778755Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojjtudpbnkdj9hn5lbovlb6qujk0 still exists (deletion dispatched at 2026-09-30T07:47:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"71787115394623ea33a3c29b5233794079ade531f8f1a6938055a3fc0f512346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"649e383ec3ed1073212333a1b5e629c51797a17f0420563d94bcc96852af6d6a","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016d82b8\u003e: \n    resource group \"admin-api-serialconsole-bootdiag-bztnlzgvbtfg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-bztnlzgvbtfg\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abff150f-53b5-4a1e-8e79-85cb6932d14c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abff150f-53b5-4a1e-8e79-85cb6932d14c\",\n      \"name\": \"abff150f-53b5-4a1e-8e79-85cb6932d14c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:44:47.122956765Z\",\n      \"endTime\": \"2026-09-30T08:08:52.492586002Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oclgsj9oapvabnc01b65dqdln9hs still exists (deletion dispatched at 2026-09-30T07:44:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016d82b8\u003e: \n    resource group \"admin-api-serialconsole-bootdiag-bztnlzgvbtfg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-bztnlzgvbtfg\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abff150f-53b5-4a1e-8e79-85cb6932d14c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/abff150f-53b5-4a1e-8e79-85cb6932d14c\",\n      \"name\": \"abff150f-53b5-4a1e-8e79-85cb6932d14c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:44:47.122956765Z\",\n      \"endTime\": \"2026-09-30T08:08:52.492586002Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56oclgsj9oapvabnc01b65dqdln9hs still exists (deletion dispatched at 2026-09-30T07:44:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"83c3d1f97626231197f625d05f64e3ad2ffd19a8b3e3e250667f61c9a171e495","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"bb57716abde8e2e547ac440d3603b828a0aa183bfdf756ba7db290fe0da391f6","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f6c198\u003e: \n    resource group \"private-kas-htntrcf5948s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-htntrcf5948s\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e211cf-72b5-460c-86b9-42f8c0ab026d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e211cf-72b5-460c-86b9-42f8c0ab026d\",\n      \"name\": \"f1e211cf-72b5-460c-86b9-42f8c0ab026d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:53:38.631153455Z\",\n      \"endTime\": \"2026-09-30T08:17:42.452924542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofmlsp87gpoiuva34g4ukacv58an still exists (deletion dispatched at 2026-09-30T07:53:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f6c198\u003e: \n    resource group \"private-kas-htntrcf5948s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-htntrcf5948s\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e211cf-72b5-460c-86b9-42f8c0ab026d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1e211cf-72b5-460c-86b9-42f8c0ab026d\",\n      \"name\": \"f1e211cf-72b5-460c-86b9-42f8c0ab026d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:53:38.631153455Z\",\n      \"endTime\": \"2026-09-30T08:17:42.452924542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofmlsp87gpoiuva34g4ukacv58an still exists (deletion dispatched at 2026-09-30T07:53:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"84e937aef87277029c42701d5c8639bd9a7d644c57a594394930765671e9d0fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"100fa41546dbaca127b035a4f15ea05b9f9ef7056f944eee271ee12155761b27","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001190090\u003e: \n    resource group \"rg-zstream-upgrade-5-0-vztfjbhwj86s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-dbc7n2\" in resourcegroup=\"rg-zstream-upgrade-5-0-vztfjbhwj86s\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d21c47-4deb-485c-adc0-8dba2db04e85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d21c47-4deb-485c-adc0-8dba2db04e85\",\n      \"name\": \"86d21c47-4deb-485c-adc0-8dba2db04e85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:00:45.311282131Z\",\n      \"endTime\": \"2026-09-30T08:24:52.368269792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojo9ccai43f073bp4dpipm3b14ma still exists (deletion dispatched at 2026-09-30T08:01:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001190090\u003e: \n    resource group \"rg-zstream-upgrade-5-0-vztfjbhwj86s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-dbc7n2\" in resourcegroup=\"rg-zstream-upgrade-5-0-vztfjbhwj86s\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d21c47-4deb-485c-adc0-8dba2db04e85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d21c47-4deb-485c-adc0-8dba2db04e85\",\n      \"name\": \"86d21c47-4deb-485c-adc0-8dba2db04e85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:00:45.311282131Z\",\n      \"endTime\": \"2026-09-30T08:24:52.368269792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojo9ccai43f073bp4dpipm3b14ma still exists (deletion dispatched at 2026-09-30T08:01:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a922694c8efe1074ae2aab65bd190ca080ec64f4ffcc72fca76326b405c8994","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"3850893a2be376a4c082764baaa2865ab6202df69d12b07798777716859179b2","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T09:10:27Z\n  Ended:       2026-09-30T10:12:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t56ofmlsp87gpoiuva34g4ukacv58an-private-kas\", region=\"westus3\", severity=\"warning\"\n  Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2t56ofmlsp87gpoiuva34g4ukacv58an-private-kas (management cluster ci01-j0283264-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}},{"row_id":"8fa4a6f00e1055bf61cd1a7994677e75d1b7029d0ae06f8b985bc51382a8ea37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"765cb31fbe9731b40edd3678a0fe26d7a83e359d9cc7b2b09560b2f8ec5e71aa","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:409 +0x87 fp=0xc0000d9e90 sp=0xc0000d9e20 pc=0xbff15a7\ngithub.com/Azure/ARO-HCP/test/util/framework.runResourceGroupCleanup.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:410 +0x7e fp=0xc0000d9f78 sp=0xc0000d9e90 pc=0xbff12be\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50 fp=0xc0000d9fe0 sp=0xc0000d9f78 pc=0x6f51610\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000d9fe8 sp=0xc0000d9fe0 pc=0x5439fc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 3248\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 3283 gp=0xc0011bfdc0 m=nil [select, 2 minutes]:\nruntime.gopark(0xc000e65f48?, 0x2?, 0xc0?, 0x5d?, 0xc000e65ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000df4d78 sp=0xc000df4d58 pc=0x54317ae\nruntime.selectgo(0xc000df4f48, 0xc000e65ef0, 0xc00191c0c0?, 0x0, 0xc0028017d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000df4eb8 sp=0xc000df4d78 pc=0x540c3b7\nnet/http.(*persistConn).writeLoop(0xc0013e3b00)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000df4fc8 sp=0xc000df4eb8 pc=0x5803227\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000df4fe0 sp=0xc000df4fc8 pc=0x57ffec5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000df4fe8 sp=0xc000df4fe0 pc=0x5439fc1\ncreated by net/http.(*Transport).dialConn in goroutine 3249\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 3232 gp=0xc0002616c0 m=nil [select]:\nruntime.gopark(0xc001a7f768?, 0x2?, 0x40?, 0x0?, 0xc001a7f75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001a7f5e8 sp=0xc001a7f5c8 pc=0x54317ae\nruntime.selectgo(0xc001a7f768, 0xc001a7f758, 0xc001a7f788?, 0x0, 0xf83b8ce?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc001a7f728 sp=0xc001a7f5e8 pc=0x540c3b7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x108bfe40, 0xc00066ad20}, 0xf7fc714?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.23.0/internal/shared/shared.go:36 +0x76 fp=0xc001a7f798 sp=0xc001a7f728 pc=0x5c1c996\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x10919ee0, {0x108bfe40, 0xc00066ad20}, 0x542ed93)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.23.0/runtime/poller.go:284 +0x732 fp=0xc001a7fa98 sp=0xc001a7f798 pc=0x5e69a92\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x108bfe40, 0xc00066ad20}, 0xc00104bae8, {0xc000f74aa0, 0x1c}, {0xc0018c1ef0, 0x15})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:591 +0x365 fp=0xc001a7fc20 sp=0xc001a7fa98 pc=0xbfaf265\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x108bfe40?, 0xc00066ad20?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:628 +0x72 fp=0xc001a7fcc8 sp=0xc001a7fc20 pc=0xbfafa12\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x108bfe40?, 0xc00066ad20?}, 0xc00066ad30?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc001a7fd20 sp=0xc001a7fcc8 pc=0x5bd1a22\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x108bfe40, 0xc00066ad20}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc001a7fe68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc001a7fd98 sp=0xc001a7fd20 pc=0x5bcface\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x108bfdd0, 0xc000f19220}, 0xc00104bae8, {0xc000f74aa0, 0x1c}, {0xc0018c1ef0, 0x15}, 0x15d3ef79800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:626 +0x285 fp=0xc001a7fef0 sp=0xc001a7fd98 pc=0xbfaf785\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:799 +0x7e fp=0xc001a7ff78 sp=0xc001a7fef0 pc=0xbfb219e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50 fp=0xc001a7ffe0 sp=0xc001a7ff78 pc=0x6f51610\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001a7ffe8 sp=0xc001a7ffe0 pc=0x5439fc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 3273\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x543bde3\nrdx    0x0\nrdi    0x169679d0\nrsi    0x80\nrbp    0x7ffc297c0df0\nrsp    0x7ffc297c0da8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x3\nr13    0x1\nr14    0x16965440\nr15    0x5\nrip    0x543bde1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\napp-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:409 +0x87 fp=0xc0000d9e90 sp=0xc0000d9e20 pc=0xbff15a7\ngithub.com/Azure/ARO-HCP/test/util/framework.runResourceGroupCleanup.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:410 +0x7e fp=0xc0000d9f78 sp=0xc0000d9e90 pc=0xbff12be\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50 fp=0xc0000d9fe0 sp=0xc0000d9f78 pc=0x6f51610\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000d9fe8 sp=0xc0000d9fe0 pc=0x5439fc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 3248\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\n\ngoroutine 3283 gp=0xc0011bfdc0 m=nil [select, 2 minutes]:\nruntime.gopark(0xc000e65f48?, 0x2?, 0xc0?, 0x5d?, 0xc000e65ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000df4d78 sp=0xc000df4d58 pc=0x54317ae\nruntime.selectgo(0xc000df4f48, 0xc000e65ef0, 0xc00191c0c0?, 0x0, 0xc0028017d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000df4eb8 sp=0xc000df4d78 pc=0x540c3b7\nnet/http.(*persistConn).writeLoop(0xc0013e3b00)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000df4fc8 sp=0xc000df4eb8 pc=0x5803227\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000df4fe0 sp=0xc000df4fc8 pc=0x57ffec5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000df4fe8 sp=0xc000df4fe0 pc=0x5439fc1\ncreated by net/http.(*Transport).dialConn in goroutine 3249\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 3232 gp=0xc0002616c0 m=nil [select]:\nruntime.gopark(0xc001a7f768?, 0x2?, 0x40?, 0x0?, 0xc001a7f75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001a7f5e8 sp=0xc001a7f5c8 pc=0x54317ae\nruntime.selectgo(0xc001a7f768, 0xc001a7f758, 0xc001a7f788?, 0x0, 0xf83b8ce?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc001a7f728 sp=0xc001a7f5e8 pc=0x540c3b7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x108bfe40, 0xc00066ad20}, 0xf7fc714?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.23.0/internal/shared/shared.go:36 +0x76 fp=0xc001a7f798 sp=0xc001a7f728 pc=0x5c1c996\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x10919ee0, {0x108bfe40, 0xc00066ad20}, 0x542ed93)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.23.0/runtime/poller.go:284 +0x732 fp=0xc001a7fa98 sp=0xc001a7f798 pc=0x5e69a92\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x108bfe40, 0xc00066ad20}, 0xc00104bae8, {0xc000f74aa0, 0x1c}, {0xc0018c1ef0, 0x15})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:591 +0x365 fp=0xc001a7fc20 sp=0xc001a7fa98 pc=0xbfaf265\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x108bfe40?, 0xc00066ad20?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:628 +0x72 fp=0xc001a7fcc8 sp=0xc001a7fc20 pc=0xbfafa12\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x108bfe40?, 0xc00066ad20?}, 0xc00066ad30?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc001a7fd20 sp=0xc001a7fcc8 pc=0x5bd1a22\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x108bfe40, 0xc00066ad20}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc001a7fe68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc001a7fd98 sp=0xc001a7fd20 pc=0x5bcface\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x108bfdd0, 0xc000f19220}, 0xc00104bae8, {0xc000f74aa0, 0x1c}, {0xc0018c1ef0, 0x15}, 0x15d3ef79800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:626 +0x285 fp=0xc001a7fef0 sp=0xc001a7fd98 pc=0xbfaf785\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/helpers_v20240610preview.go:799 +0x7e fp=0xc001a7ff78 sp=0xc001a7fef0 pc=0xbfb219e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50 fp=0xc001a7ffe0 sp=0xc001a7ff78 pc=0x6f51610\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001a7ffe8 sp=0xc001a7ffe0 pc=0x5439fc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 3273\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x543bde3\nrdx    0x0\nrdi    0x169679d0\nrsi    0x80\nrbp    0x7ffc297c0df0\nrsp    0x7ffc297c0da8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x3\nr13    0x1\nr14    0x16965440\nr15    0x5\nrip    0x543bde1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"92c5d4ff8e65eb56ab451a54560d9e78da13db079747c61455c2666cc78d0dc1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"ec90d4af2927302761e0ebd2e1983af136cd63955ff9ce58162405d9f1e62c70","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011781c8\u003e: \n    resource group \"idms-6clt25jfd748\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6clt25jfd748\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\",\n      \"name\": \"95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:43:50.391649068Z\",\n      \"endTime\": \"2026-09-30T08:07:52.519744698Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofblporgmut8brildeurh8bqh0jm still exists (deletion dispatched at 2026-09-30T07:44:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011781c8\u003e: \n    resource group \"idms-6clt25jfd748\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6clt25jfd748\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\",\n      \"name\": \"95e22e88-1144-4d00-8cc9-dc9f1a6c3ffb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:43:50.391649068Z\",\n      \"endTime\": \"2026-09-30T08:07:52.519744698Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ofblporgmut8brildeurh8bqh0jm still exists (deletion dispatched at 2026-09-30T07:44:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a63d6d9e21a8784c60c9b1367f823beff83c1a3008f7bc7766943d7ddc6e6bf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"00b952975e25b271682c84963b4e373b62836b85fd377a0a370a32c2e1265937","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001702090\u003e: \n    resource group \"fips-enabled-xs9fxbdgnwhz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xs9fxbdgnwhz\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81d3effe-1da4-43c5-8596-8a54252ca278\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81d3effe-1da4-43c5-8596-8a54252ca278\",\n      \"name\": \"81d3effe-1da4-43c5-8596-8a54252ca278\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:45:23.642226584Z\",\n      \"endTime\": \"2026-09-30T08:09:32.307764104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obh88cddik6iio0cu4ci35j02k7f still exists (deletion dispatched at 2026-09-30T07:45:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001702090\u003e: \n    resource group \"fips-enabled-xs9fxbdgnwhz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xs9fxbdgnwhz\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81d3effe-1da4-43c5-8596-8a54252ca278\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81d3effe-1da4-43c5-8596-8a54252ca278\",\n      \"name\": \"81d3effe-1da4-43c5-8596-8a54252ca278\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:45:23.642226584Z\",\n      \"endTime\": \"2026-09-30T08:09:32.307764104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obh88cddik6iio0cu4ci35j02k7f still exists (deletion dispatched at 2026-09-30T07:45:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a76da1600985c76f3b9d52aa3f285b7c1e8529df9ac813b7c1bce004ff5b8ab9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"7456f43c64352e2292d78935707ba348a1cd19ec7dd8562f66846dacd59a897b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d94390\u003e: \n    resource group \"rg-candidate-4-22-7rr572k8p4nx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hqg9nr\" in resourcegroup=\"rg-candidate-4-22-7rr572k8p4nx\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5146bdc-a88e-43b4-ade8-4264fc68118b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5146bdc-a88e-43b4-ade8-4264fc68118b\",\n      \"name\": \"c5146bdc-a88e-43b4-ade8-4264fc68118b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:52:17.26155411Z\",\n      \"endTime\": \"2026-09-30T08:16:22.313143273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojq845tseueb8p9jb82rbbpq2mbb still exists (deletion dispatched at 2026-09-30T07:52:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d94390\u003e: \n    resource group \"rg-candidate-4-22-7rr572k8p4nx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hqg9nr\" in resourcegroup=\"rg-candidate-4-22-7rr572k8p4nx\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5146bdc-a88e-43b4-ade8-4264fc68118b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5146bdc-a88e-43b4-ade8-4264fc68118b\",\n      \"name\": \"c5146bdc-a88e-43b4-ade8-4264fc68118b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:52:17.26155411Z\",\n      \"endTime\": \"2026-09-30T08:16:22.313143273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojq845tseueb8p9jb82rbbpq2mbb still exists (deletion dispatched at 2026-09-30T07:52:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a7877d69e35d62cecdfcd0dc96c267eac18b1aa045aee005c9023c988d2e9185","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"e4780041957a895982016aa7b2489f3353971e88275784b941e3e61d6674b776","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f38108\u003e: \n    resource group \"v20260901preview-fphhhngljlhp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-fphhhngljlhp\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc213230-cecc-45fc-a51e-9966fc3f25a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc213230-cecc-45fc-a51e-9966fc3f25a4\",\n      \"name\": \"dc213230-cecc-45fc-a51e-9966fc3f25a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:49.418893525Z\",\n      \"endTime\": \"2026-09-30T08:13:52.389241283Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ocsot5cpua59vgr84hkknkr96gf6 still exists (deletion dispatched at 2026-09-30T07:49:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f38108\u003e: \n    resource group \"v20260901preview-fphhhngljlhp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-fphhhngljlhp\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc213230-cecc-45fc-a51e-9966fc3f25a4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc213230-cecc-45fc-a51e-9966fc3f25a4\",\n      \"name\": \"dc213230-cecc-45fc-a51e-9966fc3f25a4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:49.418893525Z\",\n      \"endTime\": \"2026-09-30T08:13:52.389241283Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ocsot5cpua59vgr84hkknkr96gf6 still exists (deletion dispatched at 2026-09-30T07:49:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"afca035b66214935e7fe44a095fe520dae014c836b3128601d5cf587b8be842b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"7bf451011d3828ddbe210bbc04c6069d4b08635a16722de7eac25ed2398705ec","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001396090\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-23-wm46s5gt65dl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-pg8svf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-wm46s5gt65dl\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\",\n      \"name\": \"4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:57:46.367143777Z\",\n      \"endTime\": \"2026-09-30T08:21:52.387303059Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odfbmlrdpspcs2jrvj7ok5eocb47 still exists (deletion dispatched at 2026-09-30T07:57:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001396090\u003e: \n    resource group \"rg-cp-ystream-upgrade-4-23-wm46s5gt65dl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-pg8svf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-wm46s5gt65dl\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\",\n      \"name\": \"4f33f8f6-72a9-44ff-8c84-6b45107fd2f9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:57:46.367143777Z\",\n      \"endTime\": \"2026-09-30T08:21:52.387303059Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odfbmlrdpspcs2jrvj7ok5eocb47 still exists (deletion dispatched at 2026-09-30T07:57:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b6eee3cc0e2a3ca0f213841293541de2a56f897907a0f151f8582adabb33c3ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"7a693951723740214ffa37293275209a959529bb537f41e30e9db140a02cecbd","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 16 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:36:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:44:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:46:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:10:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:10:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:10:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T09:10:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mqkvs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ba89909be147865be314b6b374fbf189231f9019af30fb98657cb24dd2dfd553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"a1e6e6a948237d5895cda86936bae09d2fd958c75bf3854eb19fbf06516789a2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:301]: failed to update cluster with new KMS key with the second rotation\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfb3e0\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-vxcvln6bjmck\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-vxcvln6bjmck, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:301]: failed to update cluster with new KMS key with the second rotation\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfb3e0\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-vxcvln6bjmck\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-vxcvln6bjmck, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-vxcvln6bjmck\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-vxcvln6bjmck, error: context deadline exceeded"}},{"row_id":"c0d2df83ea88a2c469daf30d1c69a1ccfb2c026abd14cb08e2615935c77108d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"5446c05e3ffbb0d0281fc86603f62bdf8f4380f4ed38b1703787e9b1478ab704","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ec0d8\u003e: \n    resource group \"rg-zstream-upgrade-4-23-gq8tt4j82btd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-23-jjk68g\" in resourcegroup=\"rg-zstream-upgrade-4-23-gq8tt4j82btd\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/259e7251-93cc-4a54-a228-5ab61010d78f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/259e7251-93cc-4a54-a228-5ab61010d78f\",\n      \"name\": \"259e7251-93cc-4a54-a228-5ab61010d78f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:02:15.27366157Z\",\n      \"endTime\": \"2026-09-30T08:26:22.388447131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ogva7br99jqa1217jecnq155d1ej still exists (deletion dispatched at 2026-09-30T08:02:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019ec0d8\u003e: \n    resource group \"rg-zstream-upgrade-4-23-gq8tt4j82btd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-23-jjk68g\" in resourcegroup=\"rg-zstream-upgrade-4-23-gq8tt4j82btd\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/259e7251-93cc-4a54-a228-5ab61010d78f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/259e7251-93cc-4a54-a228-5ab61010d78f\",\n      \"name\": \"259e7251-93cc-4a54-a228-5ab61010d78f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:02:15.27366157Z\",\n      \"endTime\": \"2026-09-30T08:26:22.388447131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ogva7br99jqa1217jecnq155d1ej still exists (deletion dispatched at 2026-09-30T08:02:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cb0af13c1d3f394e303987bfff96c8c8cd887624d6a2e25d08c2d7e6e0eb0287","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"ff1be5ecc56cbed8526bc0d055a5d690c6803444053e4d055eb54c8a29112a59","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-30T09:05:33Z\n  Ended:       2026-09-30T09:25:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0283264-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.44:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/services-j0283264\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-8bc58bd6b-kmgbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"kubernetesevents\"\n  Description: Kusto log data for table kubernetesevents on cluster ci01-j0283264-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-30T09:06:32Z\n  Ended:       2026-09-30T09:26:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0283264-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.44:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/services-j0283264\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-8bc58bd6b-kmgbh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"kubernetesevents\"\n  Description: Kusto log data for table kubernetesevents on cluster ci01-j0283264-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}},{"row_id":"cd7ddf8a193262151390010254ef02d9d4a5a229be543633399c91f1fbf78570","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"1cfe895636607d50e1837e523cbd8ebfebbeb7a06dfe051208e969612f8d36ce","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010642a0\u003e: \n    resource group \"external-auth-cluster-4nlpdcdjkq8m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4nlpdcdjkq8m\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f11d658-22eb-436f-a6d0-266fcfe0ad04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f11d658-22eb-436f-a6d0-266fcfe0ad04\",\n      \"name\": \"0f11d658-22eb-436f-a6d0-266fcfe0ad04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:50:30.578221498Z\",\n      \"endTime\": \"2026-09-30T08:14:32.551976542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odm3uljbr1bc72mqo3pd7c5dcrar still exists (deletion dispatched at 2026-09-30T07:50:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010642a0\u003e: \n    resource group \"external-auth-cluster-4nlpdcdjkq8m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-4nlpdcdjkq8m\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f11d658-22eb-436f-a6d0-266fcfe0ad04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0f11d658-22eb-436f-a6d0-266fcfe0ad04\",\n      \"name\": \"0f11d658-22eb-436f-a6d0-266fcfe0ad04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:50:30.578221498Z\",\n      \"endTime\": \"2026-09-30T08:14:32.551976542Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56odm3uljbr1bc72mqo3pd7c5dcrar still exists (deletion dispatched at 2026-09-30T07:50:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d91756101f8b4172ed4dd117077600e0a6bd676b3c37b8f67fd945deac25a667","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"alert","signature_id":"00abdede94e48868df7e3ac1321fbb792324089549a037edfb3ae75e29326212","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 32 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"cloud-network-config-controller-6b7699b59d-mtvjg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1aa0eaea-3879-4a75-b74f-75ef531c3b1d\"\n  Description: Pod ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/cloud-network-config-controller-6b7699b59d-mtvjg (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"cloud-network-config-controller-c9f8f4988-98sd7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1b3a2719-6e7a-4a5b-aeae-d318e5e4d17e\"\n  Description: Pod ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/cloud-network-config-controller-c9f8f4988-98sd7 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"csi-snapshot-controller-ff4fdf4ff-8fn8s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"523581cf-5012-43ac-a402-c7a0a8acaac4\"\n  Description: Pod ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/csi-snapshot-controller-ff4fdf4ff-8fn8s (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"cloud-network-config-controller-5c94ccc4b5-cg2cp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"dafd5d14-e303-47e0-b4e1-d01b00ea97f1\"\n  Description: Pod ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/cloud-network-config-controller-5c94ccc4b5-cg2cp (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"cloud-network-config-controller-6b7699b59d-mtvjg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1aa0eaea-3879-4a75-b74f-75ef531c3b1d\"\n  Description: Pod ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/cloud-network-config-controller-6b7699b59d-mtvjg (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"csi-snapshot-controller-7477f9489c-8hwvb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1a334069-0f8a-44c9-ab3a-5524390a74c8\"\n  Description: Pod ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/csi-snapshot-controller-7477f9489c-8hwvb (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"cloud-network-config-controller-749db5479f-29nst\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ad67988d-3f68-4e02-84b2-957b828c23ea\"\n  Description: Pod ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/cloud-network-config-controller-749db5479f-29nst (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"cloud-network-config-controller-79d7c84c4-jpmxx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"350fe4f2-fba9-41ad-825a-5166b3b5b8f2\"\n  Description: Pod ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/cloud-network-config-controller-79d7c84c4-jpmxx (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"csi-snapshot-controller-5cc4dd7c9d-2tkdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4766f6e2-d1bc-40e5-9418-ada111a189ef\"\n  Description: Pod ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/csi-snapshot-controller-5cc4dd7c9d-2tkdw (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"csi-snapshot-controller-6445b45cbd-g46qp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4c46f8cc-e7a5-456d-88e9-4d97f220cbc0\"\n  Description: Pod ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/csi-snapshot-controller-6445b45cbd-g46qp (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:54:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"csi-snapshot-controller-68f64f94cf-f2r4v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e4f7cb9e-9e38-408b-b971-4b23a9e6fc0e\"\n  Description: Pod ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/csi-snapshot-controller-68f64f94cf-f2r4v (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"cloud-network-config-controller-c9f8f4988-98sd7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1b3a2719-6e7a-4a5b-aeae-d318e5e4d17e\"\n  Description: Pod ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/cloud-network-config-controller-c9f8f4988-98sd7 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"csi-snapshot-controller-7cf66cc49d-fnzp4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6f42eca0-0eb8-412d-9fbb-e74e80100964\"\n  Description: Pod ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/csi-snapshot-controller-7cf66cc49d-fnzp4 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"csi-snapshot-controller-6445b45cbd-g46qp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4c46f8cc-e7a5-456d-88e9-4d97f220cbc0\"\n  Description: Pod ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/csi-snapshot-controller-6445b45cbd-g46qp (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"csi-snapshot-controller-ff4fdf4ff-8fn8s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"523581cf-5012-43ac-a402-c7a0a8acaac4\"\n  Description: Pod ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/csi-snapshot-controller-ff4fdf4ff-8fn8s (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"cloud-network-config-controller-85b777dd49-s25s6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"7f4bec03-7c59-4bee-bd1a-b303f9184e5c\"\n  Description: Pod ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/cloud-network-config-controller-85b777dd49-s25s6 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:55:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"cloud-network-config-controller-79d7c84c4-jpmxx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"350fe4f2-fba9-41ad-825a-5166b3b5b8f2\"\n  Description: Pod ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/cloud-network-config-controller-79d7c84c4-jpmxx (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"csi-snapshot-controller-7cf66cc49d-fnzp4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6f42eca0-0eb8-412d-9fbb-e74e80100964\"\n  Description: Pod ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/csi-snapshot-controller-7cf66cc49d-fnzp4 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"cloud-network-config-controller-85b777dd49-s25s6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"7f4bec03-7c59-4bee-bd1a-b303f9184e5c\"\n  Description: Pod ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/cloud-network-config-controller-85b777dd49-s25s6 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"cloud-network-config-controller-c48d57456-gkqfz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"162d1556-afda-4dc1-97da-7279eda02fab\"\n  Description: Pod ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/cloud-network-config-controller-c48d57456-gkqfz (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"cloud-network-config-controller-749db5479f-29nst\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ad67988d-3f68-4e02-84b2-957b828c23ea\"\n  Description: Pod ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/cloud-network-config-controller-749db5479f-29nst (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"cloud-network-config-controller-5687d9df7d-mc789\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c80807d8-f734-41fa-bf72-fa4544a1127f\"\n  Description: Pod ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/cloud-network-config-controller-5687d9df7d-mc789 (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"csi-snapshot-controller-546555d6dd-whrqm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6e800052-3ed4-4832-ac6e-ba3a34c7eda1\"\n  Description: Pod ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/csi-snapshot-controller-546555d6dd-whrqm (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"csi-snapshot-controller-68698d756-2sjcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ea8389c4-6821-4c53-af9f-460b56d932d9\"\n  Description: Pod ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/csi-snapshot-controller-68698d756-2sjcb (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b\", pod=\"cloud-network-config-controller-c48d57456-gkqfz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"162d1556-afda-4dc1-97da-7279eda02fab\"\n  Description: Pod ocm-arohcpci01-2t56ofgpftr9snm78jc1alvisvlihdkq-z9j9b2e5q2j7y1b/cloud-network-config-controller-c48d57456-gkqfz (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o\", pod=\"csi-snapshot-controller-5cc4dd7c9d-2tkdw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4766f6e2-d1bc-40e5-9418-ada111a189ef\"\n  Description: Pod ocm-arohcpci01-2t56onftd04005nl721veuiicq8a5s6u-p1f3j3e2l1z4c9o/csi-snapshot-controller-5cc4dd7c9d-2tkdw (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk\", pod=\"csi-snapshot-controller-68698d756-2sjcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ea8389c4-6821-4c53-af9f-460b56d932d9\"\n  Description: Pod ocm-arohcpci01-2t56ok2d2baqjscldo4vmovim4i18pgh-ephemeral-disk/csi-snapshot-controller-68698d756-2sjcb (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster\", pod=\"csi-snapshot-controller-68f64f94cf-f2r4v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e4f7cb9e-9e38-408b-b971-4b23a9e6fc0e\"\n  Description: Pod ocm-arohcpci01-2t56oguqit52ovjjoaakam9v60fven2c-sre-hcp-cluster/csi-snapshot-controller-68f64f94cf-f2r4v (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:56:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h\", pod=\"csi-snapshot-controller-546555d6dd-whrqm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6e800052-3ed4-4832-ac6e-ba3a34c7eda1\"\n  Description: Pod ocm-arohcpci01-2t56ofblporgmut8brildeurh8bqh0jm-b8a2i8y2x4y0w5h/csi-snapshot-controller-546555d6dd-whrqm (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:57:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y\", pod=\"csi-snapshot-controller-7477f9489c-8hwvb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1a334069-0f8a-44c9-ab3a-5524390a74c8\"\n  Description: Pod ocm-arohcpci01-2t56obh88cddik6iio0cu4ci35j02k7f-f1s2k3l8b2h6n5y/csi-snapshot-controller-7477f9489c-8hwvb (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 31:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:57:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29\", pod=\"cloud-network-config-controller-5c94ccc4b5-cg2cp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"dafd5d14-e303-47e0-b4e1-d01b00ea97f1\"\n  Description: Pod ocm-arohcpci01-2t56p42at8ondug5mpb3mmh6f9hh71ck-ea-listrk7c29/cloud-network-config-controller-5c94ccc4b5-cg2cp (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 32:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T08:57:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0283264-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0283264/providers/microsoft.monitor/accounts/hcps-j0283264\", namespace=\"ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b\", pod=\"cloud-network-config-controller-5687d9df7d-mc789\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c80807d8-f734-41fa-bf72-fa4544a1127f\"\n  Description: Pod ocm-arohcpci01-2t56oclgsj9oapvabnc01b65dqdln9hs-q5s4p4g3c9i1p1b/cloud-network-config-controller-5687d9df7d-mc789 (controller) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"dc1e25651421cdafde7ab4da463caef1ab2619d192be0d1a8d833f6246d3935a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"702288a89ce7dc61e45693f6260c580c01e0f296f382f69e2dd561371dbfa066","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cd63f0\u003e: \n    resource group \"arm64-vm-cluster-62xw9fcdx42j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-62xw9fcdx42j\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d160b43-5f34-4632-80cb-5de8f2fd3f41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d160b43-5f34-4632-80cb-5de8f2fd3f41\",\n      \"name\": \"7d160b43-5f34-4632-80cb-5de8f2fd3f41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:46:16.079175081Z\",\n      \"endTime\": \"2026-09-30T08:10:22.378932829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ocn9um98ldcojp1pl53iclpb0kl7 still exists (deletion dispatched at 2026-09-30T07:46:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cd63f0\u003e: \n    resource group \"arm64-vm-cluster-62xw9fcdx42j\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-62xw9fcdx42j\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d160b43-5f34-4632-80cb-5de8f2fd3f41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7d160b43-5f34-4632-80cb-5de8f2fd3f41\",\n      \"name\": \"7d160b43-5f34-4632-80cb-5de8f2fd3f41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:46:16.079175081Z\",\n      \"endTime\": \"2026-09-30T08:10:22.378932829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ocn9um98ldcojp1pl53iclpb0kl7 still exists (deletion dispatched at 2026-09-30T07:46:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e8da205e9aa80dcc7bf092793526ee503f73e2df50f20bd4637994d3a6dbc951","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"e0bad5b53413f07d5e84ab7f0e9ceb6826abb2c93e45e9f508ec37a3532918d0","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012cc090\u003e: \n    resource group \"ea-list-tcgjz7swjvdw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listrk7c29\" in resourcegroup=\"ea-list-tcgjz7swjvdw\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19fe26bb-4a1e-4ebc-baa6-f57c53e83296\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19fe26bb-4a1e-4ebc-baa6-f57c53e83296\",\n      \"name\": \"19fe26bb-4a1e-4ebc-baa6-f57c53e83296\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:43:53.484653514Z\",\n      \"endTime\": \"2026-09-30T08:08:02.489234786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56p42at8ondug5mpb3mmh6f9hh71ck still exists (deletion dispatched at 2026-09-30T07:44:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012cc090\u003e: \n    resource group \"ea-list-tcgjz7swjvdw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listrk7c29\" in resourcegroup=\"ea-list-tcgjz7swjvdw\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19fe26bb-4a1e-4ebc-baa6-f57c53e83296\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19fe26bb-4a1e-4ebc-baa6-f57c53e83296\",\n      \"name\": \"19fe26bb-4a1e-4ebc-baa6-f57c53e83296\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:43:53.484653514Z\",\n      \"endTime\": \"2026-09-30T08:08:02.489234786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56p42at8ondug5mpb3mmh6f9hh71ck still exists (deletion dispatched at 2026-09-30T07:44:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f44934dcd5dc7f5bed73669d671ede48170369c32f258b980f9017ea4e6e7399","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"f9a387d86ae70031b56c0d07a7a2eac9fb255e096d9d3c876eca81655334979d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0005468d0\u003e: \n    resource group \"rg-candidate-4-20-6zgdjl96fd2x\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-dn6n8z\" in resourcegroup=\"rg-candidate-4-20-6zgdjl96fd2x\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ec571ff-78ed-435f-b6c6-33518aa00501\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ec571ff-78ed-435f-b6c6-33518aa00501\",\n      \"name\": \"3ec571ff-78ed-435f-b6c6-33518aa00501\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:55:49.208034763Z\",\n      \"endTime\": \"2026-09-30T08:19:52.415025374Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojtndhilnoii39q1v1t13gbjb4ah still exists (deletion dispatched at 2026-09-30T07:55:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0005468d0\u003e: \n    resource group \"rg-candidate-4-20-6zgdjl96fd2x\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-dn6n8z\" in resourcegroup=\"rg-candidate-4-20-6zgdjl96fd2x\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ec571ff-78ed-435f-b6c6-33518aa00501\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ec571ff-78ed-435f-b6c6-33518aa00501\",\n      \"name\": \"3ec571ff-78ed-435f-b6c6-33518aa00501\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:55:49.208034763Z\",\n      \"endTime\": \"2026-09-30T08:19:52.415025374Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56ojtndhilnoii39q1v1t13gbjb4ah still exists (deletion dispatched at 2026-09-30T07:55:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f897c9adcc78f1e14a46b32c6431b2604a26ec3242433675cf0fb2e36d2d47fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7261/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105187789510283264","occurred_at":"2026-09-30T06:47:34Z","failed_at":"e2e","signature_id":"92fe7c553dcdfef3768979dc519edf0e7995e69a064128d00e345d263790f82e","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001ed0138\u003e: \n    resource group \"rg-hs-pre--7p7rhfskdkg8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-mxbwvc\" in resourcegroup=\"rg-hs-pre--7p7rhfskdkg8\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\",\n      \"name\": \"38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:12.08839353Z\",\n      \"endTime\": \"2026-09-30T08:13:12.35854499Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obncvdr9t1vsvq3snfkacnvh3cke still exists (deletion dispatched at 2026-09-30T07:49:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001ed0138\u003e: \n    resource group \"rg-hs-pre--7p7rhfskdkg8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-mxbwvc\" in resourcegroup=\"rg-hs-pre--7p7rhfskdkg8\" to finish deleting: GET https://rp.j0283264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\",\n      \"name\": \"38b5c072-1d7b-4021-9dde-2bef3cf5ae9a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T07:49:12.08839353Z\",\n      \"endTime\": \"2026-09-30T08:13:12.35854499Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56obncvdr9t1vsvq3snfkacnvh3cke still exists (deletion dispatched at 2026-09-30T07:49:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":35,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1"],"matched_occurrences":35,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105186118872862720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"open","pr_sha":"96ca57cb7d5eab9eb9db1b06eaec792e6f8685b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T06:40:57Z","started_at":"2026-09-30T06:40:57Z","completed_at":"2026-09-30T08:59:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:11:41.312283573Z","timing_metadata_checked_at":"2026-09-30T09:11:41.312283573Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:11:41.316143062Z","region_metadata_checked_at":"2026-09-30T09:11:41.316143062Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105177784723705856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T06:07:50Z","started_at":"2026-09-30T06:07:50Z","completed_at":"2026-09-30T08:27:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T08:40:21.848278244Z","timing_metadata_checked_at":"2026-09-30T08:40:21.848278244Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T08:40:21.851281134Z","region_metadata_checked_at":"2026-09-30T08:40:21.851281134Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7258/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105170147533656064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7258,"pr_state":"closed","pr_sha":"fc6fe3e052e452bc85be384a80e0c231a4ac5b9b","final_merged_sha":"a68022814b830b9a8334e00ff48929e4987b9c4a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T05:37:29Z","started_at":"2026-09-30T05:37:29Z","completed_at":"2026-09-30T07:52:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T08:09:08.404045017Z","timing_metadata_checked_at":"2026-09-30T08:09:08.404045017Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T08:09:08.407668906Z","region_metadata_checked_at":"2026-09-30T08:09:08.407668906Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105167861478592512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"541a1eba3ec70c4d9c21dacb6cc888e561ca9f11","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T05:28:24Z","started_at":"2026-09-30T05:28:24Z","completed_at":"2026-09-30T07:39:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T07:58:09.691706225Z","timing_metadata_checked_at":"2026-09-30T07:58:09.691706225Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T07:58:09.694735017Z","region_metadata_checked_at":"2026-09-30T07:58:09.694735017Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4ffb183209c2b567847865a5e816fc23a41d17b1785cf246dc8ab93f2041bf84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105167861478592512","occurred_at":"2026-09-30T05:28:24Z","failed_at":"e2e","signature_id":"241350474584925c505b6db759b0c1f71619ba1d16c84a08ca9783ed5efd6581","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:487]: ClusterVersion had no completed update entry in history\nExpected\n    \u003cstring\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:487]: ClusterVersion had no completed update entry in history\nExpected\n    \u003cstring\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"579aca9a055614c4c372f82f6c1cd8ec85f037f92e539fb50ce4ea435c084257","failure_pattern":"ClusterVersion had no completed update entry in history","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:487]: ClusterVersion had no completed update entry in history"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["579aca9a055614c4c372f82f6c1cd8ec85f037f92e539fb50ce4ea435c084257"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105165316576251904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7158,"pr_state":"open","pr_sha":"ad4b8d135c2f719ba56447c9417bd2d0c51dbf22","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-30T05:18:18Z","started_at":"2026-09-30T05:18:18Z","completed_at":"2026-09-30T07:34:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T07:47:14.865476292Z","timing_metadata_checked_at":"2026-09-30T07:47:14.865476292Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T07:47:14.869828678Z","region_metadata_checked_at":"2026-09-30T07:47:14.869828678Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"5c5a9007f6743cd240206942b466c26c1d9fc804d090e4a923afbab1e6608ea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105165316576251904","occurred_at":"2026-09-30T05:18:18Z","failed_at":"alert","signature_id":"0e4d81750c983a9cda845d641cdc28ac3126144f9f77408e60f4a30101f8fb9b","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:42:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:42:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T06:49:03Z\n  Ended:       2026-09-30T07:01:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55he6nad38adkphki745bqugvhhgmi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2t55he6nad38adkphki745bqugvhhgmi/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T06:49:04Z\n  Ended:       2026-09-30T07:01:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55he6nad38adkphki745bqugvhhgmi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2t55he6nad38adkphki745bqugvhhgmi/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"78f5d749ba47b252b6e96b7f7cbe61dbe94c533829e8931a0126e9dbb96a39c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105165316576251904","occurred_at":"2026-09-30T05:18:18Z","failed_at":"alert","signature_id":"f24999767c880e9e7cc09f5f63e3720ef83e3427a0a79dedbcfca71bdb5f3dae","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:07:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-78648d9c8d-czf5c\", region=\"westus3\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-78648d9c8d-czf5c has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"846903df095efb338b6f1d9ffe27dc9dfb97c4eb67a5730e4fa51bcc62c61082","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105165316576251904","occurred_at":"2026-09-30T05:18:18Z","failed_at":"alert","signature_id":"e449da77403f611870944fef6e8e8243f367e4154d6925c4af4f6d5765343341","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:32:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-30T06:32:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-30T06:39:00Z\n  Ended:       2026-09-30T07:01:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55he6nad38adkphki745bqugvhhgmi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55he6nad38adkphki745bqugvhhgmi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-30T06:39:00Z\n  Ended:       2026-09-30T07:01:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55he6nad38adkphki745bqugvhhgmi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55he6nad38adkphki745bqugvhhgmi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-30T07:02:01Z\n  Ended:       2026-09-30T07:08:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55s5ip2mfud3dnigkagbgr229suldb\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55s5ip2mfud3dnigkagbgr229suldb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-30T07:02:01Z\n  Ended:       2026-09-30T07:08:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55sahljftd8u6q1m4sbae73q1qo4f2\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55sahljftd8u6q1m4sbae73q1qo4f2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-30T07:02:01Z\n  Ended:       2026-09-30T07:08:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55s5ip2mfud3dnigkagbgr229suldb\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55s5ip2mfud3dnigkagbgr229suldb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-30T07:02:01Z\n  Ended:       2026-09-30T07:08:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55sahljftd8u6q1m4sbae73q1qo4f2\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55sahljftd8u6q1m4sbae73q1qo4f2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-30T07:03:02Z\n  Ended:       2026-09-30T07:12:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55soh6c16uoks7fn5rqknhr3g6cc88\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55soh6c16uoks7fn5rqknhr3g6cc88/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-30T07:03:02Z\n  Ended:       2026-09-30T07:12:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55soh6c16uoks7fn5rqknhr3g6cc88\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55soh6c16uoks7fn5rqknhr3g6cc88/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-30T07:03:02Z\n  Ended:       2026-09-30T07:11:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55som67oi1sqntaq9acgv9gavh7vjf\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55som67oi1sqntaq9acgv9gavh7vjf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-30T07:03:02Z\n  Ended:       2026-09-30T07:11:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6251904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6251904/providers/microsoft.monitor/accounts/services-j6251904\", namespace=\"klusterlet-2t55som67oi1sqntaq9acgv9gavh7vjf\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-blwss\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t55som67oi1sqntaq9acgv9gavh7vjf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105159944369082368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7229,"pr_state":"closed","pr_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","final_merged_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T04:56:57Z","started_at":"2026-09-30T04:56:57Z","completed_at":"2026-09-30T07:11:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T07:26:23.763459131Z","timing_metadata_checked_at":"2026-09-30T07:26:23.763459131Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T07:26:23.766679221Z","region_metadata_checked_at":"2026-09-30T07:26:23.766679221Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"08692a2a9bb993dc523ebc6bad950528d9a4e4e132cf9f5c58cb36f896caa8a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105159944369082368","occurred_at":"2026-09-30T04:56:57Z","failed_at":"e2e","signature_id":"b7b881f56be7fc493c98503286f3b339d22b05150a32a7c2547bdf5dad7f2da5","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-78dt7bb9qdx9\" to finish creating: GET https://rp.j9082368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f34a8c-014f-4784-8b2d-d242f6bfcb85\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f34a8c-014f-4784-8b2d-d242f6bfcb85\",\n  \"name\": \"e3f34a8c-014f-4784-8b2d-d242f6bfcb85\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-30T05:49:26.691969203Z\",\n  \"endTime\": \"2026-09-30T06:08:29.964580175Z\",\n  \"error\": {\n    \"code\": \"DeadlineExceeded\",\n    \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-78dt7bb9qdx9--managed/providers/Microsoft.Compute/virtualMachines/t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-78dt7bb9qdx9--managed/providers/Microsoft.Compute/virtualMachines/t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d to exist\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-78dt7bb9qdx9\" to finish creating: GET https://rp.j9082368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f34a8c-014f-4784-8b2d-d242f6bfcb85\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f34a8c-014f-4784-8b2d-d242f6bfcb85\",\n  \"name\": \"e3f34a8c-014f-4784-8b2d-d242f6bfcb85\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-30T05:49:26.691969203Z\",\n  \"endTime\": \"2026-09-30T06:08:29.964580175Z\",\n  \"error\": {\n    \"code\": \"DeadlineExceeded\",\n    \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-78dt7bb9qdx9--managed/providers/Microsoft.Compute/virtualMachines/t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-78dt7bb9qdx9--managed/providers/Microsoft.Compute/virtualMachines/t1n4p5j2m9y1n6b-np-update-nodes-fvh6z-s276d to exist\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105143371457630208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T03:51:06Z","started_at":"2026-09-30T03:51:06Z","completed_at":"2026-09-30T04:10:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:25:33.371744093Z","timing_metadata_checked_at":"2026-09-30T04:25:33.371744093Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-30T04:25:33.375447782Z","region_metadata_checked_at":"2026-09-30T05:28:39.014161271Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"68641540efce75138e29794b3953a507c41d24802b873009a748bc5e5e2908bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105143371457630208","occurred_at":"2026-09-30T03:51:06Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105143374678855680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7244,"pr_state":"closed","pr_sha":"cbddf39100b6fb0f90835d16a76eb4cb925cc7df","final_merged_sha":"cbddf39100b6fb0f90835d16a76eb4cb925cc7df","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T03:51:06Z","started_at":"2026-09-30T03:51:06Z","completed_at":"2026-09-30T06:11:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T06:23:57.279533434Z","timing_metadata_checked_at":"2026-09-30T06:23:57.279533434Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T06:23:57.282751024Z","region_metadata_checked_at":"2026-09-30T06:23:57.282751024Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105143304160022528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7238,"pr_state":"closed","pr_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","final_merged_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T03:50:49Z","started_at":"2026-09-30T03:50:49Z","completed_at":"2026-09-30T06:15:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T06:34:35.190711072Z","timing_metadata_checked_at":"2026-09-30T06:34:35.190711072Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T06:34:35.193966462Z","region_metadata_checked_at":"2026-09-30T06:34:35.193966462Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105132902411931648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T03:09:30Z","started_at":"2026-09-30T03:09:30Z","completed_at":"2026-09-30T05:36:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T05:49:44.6045911Z","timing_metadata_checked_at":"2026-09-30T05:49:44.6045911Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T05:49:44.60781189Z","region_metadata_checked_at":"2026-09-30T05:49:44.60781189Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"52c6504f3dce91cfdbfdd66a274d0e4263e289333775f4141281bf373e24ce03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105132902411931648","occurred_at":"2026-09-30T03:09:30Z","failed_at":"e2e","signature_id":"93fbf55b219b33dbed79b2f7d0789e2d4a2fce95865a645fcfba661c8688665a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010db2a0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gwhcb7q77s5g\" to finish creating: GET https://rp.j1931648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26eb862-dd0d-4748-bf3d-aaa27c84e026\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26eb862-dd0d-4748-bf3d-aaa27c84e026\",\n      \"name\": \"e26eb862-dd0d-4748-bf3d-aaa27c84e026\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T04:22:37.324871209Z\",\n      \"endTime\": \"2026-09-30T04:41:46.747553014Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gwhcb7q77s5g--managed/providers/Microsoft.Compute/virtualMachines/l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gwhcb7q77s5g--managed/providers/Microsoft.Compute/virtualMachines/l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010db2a0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gwhcb7q77s5g\" to finish creating: GET https://rp.j1931648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26eb862-dd0d-4748-bf3d-aaa27c84e026\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e26eb862-dd0d-4748-bf3d-aaa27c84e026\",\n      \"name\": \"e26eb862-dd0d-4748-bf3d-aaa27c84e026\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T04:22:37.324871209Z\",\n      \"endTime\": \"2026-09-30T04:41:46.747553014Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gwhcb7q77s5g--managed/providers/Microsoft.Compute/virtualMachines/l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gwhcb7q77s5g--managed/providers/Microsoft.Compute/virtualMachines/l4b9n5d1w5j2a3f-autoscale-noaz-wmbj7-vtxcx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105112031697833984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T01:46:34Z","started_at":"2026-09-30T01:46:34Z","completed_at":"2026-09-30T04:01:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:15:05.670308444Z","timing_metadata_checked_at":"2026-09-30T04:15:05.670308444Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T04:15:05.673626135Z","region_metadata_checked_at":"2026-09-30T04:15:05.673626135Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105107374309117952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7229,"pr_state":"closed","pr_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","final_merged_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T01:28:03Z","started_at":"2026-09-30T01:28:03Z","completed_at":"2026-09-30T03:46:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:04:20.674529776Z","timing_metadata_checked_at":"2026-09-30T04:04:20.674529776Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T04:04:20.67949106Z","region_metadata_checked_at":"2026-09-30T04:04:20.67949106Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"530dc82e6293a2d2ae7f02cd41370127a1f364dc8a592f851afe21243fc02284","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105107374309117952","occurred_at":"2026-09-30T01:28:03Z","failed_at":"e2e","signature_id":"a820df5e90389c4dc2a4194c6adffcc807b799a735d1bfbc28265e77991fcf6b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009224e0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-kv5cgd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-kv5cgd\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-agg-cluster-kv5cgd' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009224e0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-kv5cgd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-kv5cgd\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: DeploymentNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"DeploymentNotFound\",\n        \"message\": \"Deployment 'mi-agg-cluster-kv5cgd' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105106955633692672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"open","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-30T01:26:23Z","started_at":"2026-09-30T01:26:23Z","completed_at":"2026-09-30T03:42:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T03:53:37.014712713Z","timing_metadata_checked_at":"2026-09-30T03:53:37.014712713Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T03:53:37.0189678Z","region_metadata_checked_at":"2026-09-30T03:53:37.0189678Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105092336139636736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"closed","pr_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","final_merged_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T00:28:18Z","started_at":"2026-09-30T00:28:18Z","completed_at":"2026-09-30T02:47:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T03:01:35.899144616Z","timing_metadata_checked_at":"2026-09-30T03:01:35.899144616Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T03:01:35.902808904Z","region_metadata_checked_at":"2026-09-30T03:01:35.902808904Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"int","summary":{"total_runs":8,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":2,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T22:21:45Z","started_at":"2026-09-30T22:21:45Z","completed_at":"2026-10-01T00:24:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:36:09.266385428Z","timing_metadata_checked_at":"2026-10-01T00:36:09.266385428Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:36:09.269177919Z","region_metadata_checked_at":"2026-10-01T00:36:09.269177919Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"17af8a5e7c68d52a389b919fb920d09c2be6433813d46132d8303210c5df2249","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"e1ab80ecaa4d93580d7c4123eb9b36e5aa8ee7e8016e3b87ea8339aef183b157","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-w4nsqz\" to follow minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000d2e780\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-nhrfp5qbdv2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-w4nsqz\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-w4nsqz\" to follow minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000d2e780\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-nhrfp5qbdv2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-w4nsqz\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"1d946692f53b7ddfb71274512e8ad1a69e04328a6b36107a32ae69ee309d2072","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"feb7e71a8e25fd405d885cf41ce70033a8847e55948cf7eedde392b11122f746","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-lr7n9c\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00144e180\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-zh65jflwpg9j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-lr7n9c\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-lr7n9c\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00144e180\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-zh65jflwpg9j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-lr7n9c\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"341085a6cb45077ea96821e0595d1b85c6f61dbeae1172aec6eab8a8ba6d3182","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"521904fcc761d9cd02a2b6f95abeee0e3a392eb0ac2b47a7d93d25a55eecf08c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-nkn9cp\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000fb60c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-2s89ppfpwxxt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-nkn9cp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-nkn9cp\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000fb60c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-2s89ppfpwxxt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-nkn9cp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"3923e3765e4470a774960adbcee5a9f58086d90c2efb5e2a5f5c73ef9a846bb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"c0946200eceda37a08d59a185baab0cb980b7687739e47606107cf76ac8426b6","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-8g7jd8\" to 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0014b4e10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-c244qsxrhnbn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-8g7jd8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-8g7jd8\" to 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0014b4e10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-c244qsxrhnbn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-8g7jd8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"4cec2644e31a9ee29c849779560d1fa775ddb5cc7f49a1034964f3c45737d453","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"742c81892ba7c2548a8fe6e5e799a6a5f08c278c8e7b8f7f12b4ea46ffc5ee74","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d6f340\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-t8kp5gffh7ph\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fbb7c321-b5c6-4145-9be8-48a64282647e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fbb7c321-b5c6-4145-9be8-48a64282647e\",\n      \"name\": \"fbb7c321-b5c6-4145-9be8-48a64282647e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T22:47:02.188860165Z\",\n      \"endTime\": \"2026-09-30T23:06:03.834732494Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g1n9i9o3o7q1q9w-np-1-42h49-pdfcq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-t8kp5gffh7ph--managed/providers/Microsoft.Compute/virtualMachines/g1n9i9o3o7q1q9w-np-1-42h49-pdfcq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine g1n9i9o3o7q1q9w-np-1-42h49-pdfcq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-t8kp5gffh7ph--managed/providers/Microsoft.Compute/virtualMachines/g1n9i9o3o7q1q9w-np-1-42h49-pdfcq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d6f340\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-t8kp5gffh7ph\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fbb7c321-b5c6-4145-9be8-48a64282647e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fbb7c321-b5c6-4145-9be8-48a64282647e\",\n      \"name\": \"fbb7c321-b5c6-4145-9be8-48a64282647e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T22:47:02.188860165Z\",\n      \"endTime\": \"2026-09-30T23:06:03.834732494Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g1n9i9o3o7q1q9w-np-1-42h49-pdfcq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-t8kp5gffh7ph--managed/providers/Microsoft.Compute/virtualMachines/g1n9i9o3o7q1q9w-np-1-42h49-pdfcq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine g1n9i9o3o7q1q9w-np-1-42h49-pdfcq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-t8kp5gffh7ph--managed/providers/Microsoft.Compute/virtualMachines/g1n9i9o3o7q1q9w-np-1-42h49-pdfcq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"613bc2ff81c89a798af0558b01df1f5fb00903059922019fef1263103ee9c216","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"46df059a5c5e9df1e09f5d59ad8016cc1cb6a0093467dcbde365a5ff90189b69","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-22-52n697\" to follow minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015481e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-52jx5pj75l4v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-52n697\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-22-52n697\" to follow minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015481e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-52jx5pj75l4v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-52n697\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"a37571f6c5e7c222959c26a7db8506727889ebdb3862298d4d19d84a5d6d7530","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"767957008aa3581e0ed2bdbb42b5d5a4bb51493ab4f90cfbeabd2ed440394b4b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-j4z2m9\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c73320\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-xlhwbtlldbgk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-j4z2m9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-j4z2m9\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c73320\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-xlhwbtlldbgk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-j4z2m9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"c47c5b5c18e4306e0286a6dafaf7d3b743ab47e0ce300b8075babf13f6c6e7fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"04c6f021a676806444bfe19f34c02a730003765c8ad1e227cf3c6fa2eaeb44ee","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-20-b9ld4k\" to follow minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a93bf0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-m7g7x4rhlkmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-b9ld4k\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-20-b9ld4k\" to follow minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a93bf0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-m7g7x4rhlkmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-b9ld4k\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"cad551bbee7f7005e2ec8f214edbc70b6e3395431a0485b9e40ccd4c55b960d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"a21a4b40db90e6a50d9000d3479677549a0df689595a74da5480200d7e8306cb","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-6czr45\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00106e2a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-zs8jz58nhzb5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-6czr45\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-6czr45\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00106e2a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-zs8jz58nhzb5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-6czr45\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"ec9fc911ae92c0aef0ab91db23f2d71389c38306bc3e4a3c364152b8e078faac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"68beced747813bc5c2b7f70499b58a8b0c81b1fd83cafe43bbaad8d49564fbc3","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-d9vb4z\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f5c990\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-v5vn6xvq7g9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-d9vb4z\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-d9vb4z\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f5c990\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-v5vn6xvq7g9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-d9vb4z\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"f97db45a9e6a1202a2c2b2390fed6c3721c63f1cdf50058d5186b0373114211c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105422878328492032","occurred_at":"2026-09-30T22:21:45Z","failed_at":"e2e","signature_id":"e33e74ff5b3dde4f11fdf945eb7f244c17374cd7f5df985b6e2fa759e7d250a0","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-5-0-xw9s99\" to follow minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00095f590\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-248rf82xp8ms/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-xw9s99\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-5-0-xw9s99\" to follow minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00095f590\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-248rf82xp8ms/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-xw9s99\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105375087707295744","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T19:11:51Z","started_at":"2026-09-30T19:11:51Z","completed_at":"2026-09-30T21:18:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T21:26:54.857787356Z","timing_metadata_checked_at":"2026-09-30T21:26:54.857787356Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T21:26:54.860428047Z","region_metadata_checked_at":"2026-09-30T21:26:54.860428047Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"479f612c79dd15cad4d1b5224772932f706551212fbab20287b211e5c5ab544c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105375087707295744","occurred_at":"2026-09-30T19:11:51Z","failed_at":"e2e","signature_id":"3dfb0d11752d4f5ffcdc5dd8d77ed4c8b315ed9b9ddcd539ba7a4abcf9f40202","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088480\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-mmgvqz\" in resourcegroup=\"rg-np-version-upgrade-mmgvqz-rdcrv8s7btgd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1caf488-0088-4fa0-8a37-c1f30126d846\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1caf488-0088-4fa0-8a37-c1f30126d846\",\n      \"name\": \"b1caf488-0088-4fa0-8a37-c1f30126d846\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T20:37:05.655658972Z\",\n      \"endTime\": \"2026-09-30T20:56:13.566012243Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6u2x7j9c3m3v3g-npupgrade-4-20-vxvrm-whh28: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mmgvqz-rdcrv8s7btgd-np-upgrade-mm-8330a333/providers/Microsoft.Compute/virtualMachines/u6u2x7j9c3m3v3g-npupgrade-4-20-vxvrm-whh28 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6u2x7j9c3m3v3g-npupgrade-4-20-vxvrm-whh28: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mmgvqz-rdcrv8s7btgd-np-upgrade-mm-8330a333/providers/Microsoft.Compute/virtualMachines/u6u2x7j9c3m3v3g-npupgrade-4-20-vxvrm-whh28 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088480\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-mmgvqz\" in resourcegroup=\"rg-np-version-upgrade-mmgvqz-rdcrv8s7btgd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1caf488-0088-4fa0-8a37-c1f30126d846\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1caf488-0088-4fa0-8a37-c1f30126d846\",\n      \"name\": \"b1caf488-0088-4fa0-8a37-c1f30126d846\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T20:37:05.655658972Z\",\n      \"endTime\": \"2026-09-30T20:56:13.566012243Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6u2x7j9c3m3v3g-npupgrade-4-20-vxvrm-whh28: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mmgvqz-rdcrv8s7btgd-np-upgrade-mm-8330a333/providers/Microsoft.Compute/virtualMachines/u6u2x7j9c3m3v3g-npupgrade-4-20-vxvrm-whh28 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6u2x7j9c3m3v3g-npupgrade-4-20-vxvrm-whh28: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mmgvqz-rdcrv8s7btgd-np-upgrade-mm-8330a333/providers/Microsoft.Compute/virtualMachines/u6u2x7j9c3m3v3g-npupgrade-4-20-vxvrm-whh28 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105336380325367808","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T16:37:51Z","started_at":"2026-09-30T16:37:51Z","completed_at":"2026-09-30T19:03:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T19:12:04.191734278Z","timing_metadata_checked_at":"2026-09-30T19:12:04.191734278Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T19:12:04.19432637Z","region_metadata_checked_at":"2026-09-30T19:12:04.19432637Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2c8c12455ba4d6b1b49cb792ac0d7374b8efca10ca0c2b1d8f91da263a5d0dd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105336380325367808","occurred_at":"2026-09-30T16:37:51Z","failed_at":"e2e","signature_id":"474d961f737a5a086d6b9b16df761979aee80ac6a28d2519109fc6cfbba948ed","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-20-t74pss\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b40260\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-20-t74pss-7wvjkj\" in resourcegroup=\"rg-candidate-4-20-bmfs7fmklpfb\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-20-bmfs7fmklpfb/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-20-t74pss-7wvjkj/operationStatuses/08584108180410013833\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-cr5ick5opsuzu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-20-t74pss\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b40260\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-20-t74pss-7wvjkj\" in resourcegroup=\"rg-candidate-4-20-bmfs7fmklpfb\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-20-bmfs7fmklpfb/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-20-t74pss-7wvjkj/operationStatuses/08584108180410013833\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-cr5ick5opsuzu' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90dc413b9b32410c56e01198bcd6508fef157c78b13b631dfaa98ec6f0de2c54","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["90dc413b9b32410c56e01198bcd6508fef157c78b13b631dfaa98ec6f0de2c54"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105320733671428096","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T15:35:52Z","started_at":"2026-09-30T15:35:52Z","completed_at":"2026-09-30T15:36:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T15:52:13.407225147Z","timing_metadata_checked_at":"2026-09-30T15:52:13.407225147Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-30T15:52:13.41359483Z","region_metadata_checked_at":"2026-09-30T16:54:31.093610927Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"15564e4d1294285c7b5027ab78c9da421a79814d9b2890fdb5227264c815cf2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105320733671428096","occurred_at":"2026-09-30T15:35:52Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105317223974309888","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T15:21:55Z","started_at":"2026-09-30T15:21:55Z","completed_at":"2026-09-30T15:31:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T15:41:35.483715439Z","timing_metadata_checked_at":"2026-09-30T15:41:35.483715439Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-30T15:41:35.487172327Z","region_metadata_checked_at":"2026-09-30T16:44:13.430614165Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2db6dbcdb9508914658d7801437a87894aa012a8db0733faf77c2d5890a5c0eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105317223974309888","occurred_at":"2026-09-30T15:21:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105267578321506304","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T12:04:39Z","started_at":"2026-09-30T12:04:39Z","completed_at":"2026-09-30T14:40:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:58:38.727905845Z","timing_metadata_checked_at":"2026-09-30T14:58:38.727905845Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:58:38.731511633Z","region_metadata_checked_at":"2026-09-30T14:58:38.731511633Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105204066635223040","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T07:52:16Z","started_at":"2026-09-30T07:52:16Z","completed_at":"2026-09-30T10:21:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T10:35:01.4719969Z","timing_metadata_checked_at":"2026-09-30T10:35:01.4719969Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T10:35:01.474652392Z","region_metadata_checked_at":"2026-09-30T10:35:01.474652392Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105125675869409280","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T02:40:47Z","started_at":"2026-09-30T02:40:47Z","completed_at":"2026-09-30T05:05:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T05:18:00.120184787Z","timing_metadata_checked_at":"2026-09-30T05:18:00.120184787Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T05:18:00.123738176Z","region_metadata_checked_at":"2026-09-30T05:18:00.123738176Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":3,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T23:00:28Z","started_at":"2026-09-30T23:00:28Z","completed_at":"2026-10-01T00:52:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:57:52.604542948Z","timing_metadata_checked_at":"2026-10-01T00:57:52.604542948Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:57:52.611293127Z","region_metadata_checked_at":"2026-10-01T00:57:52.611293127Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"21ec1c31375dd75bd46c14c1e026b20cde8fffc509cfb8f0e3e177228f40a6ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"6342306aa2c9342124f24885b42800960540bcf682af76df09c7083cf26dad88","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013120e0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sttxg5n27hh5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95422086-7eb5-4fab-930f-f78b5c9185bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95422086-7eb5-4fab-930f-f78b5c9185bc\",\n      \"name\": \"95422086-7eb5-4fab-930f-f78b5c9185bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T00:05:50.759671414Z\",\n      \"endTime\": \"2026-10-01T00:24:59.018405341Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sttxg5n27hh5--managed/providers/Microsoft.Compute/virtualMachines/m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sttxg5n27hh5--managed/providers/Microsoft.Compute/virtualMachines/m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013120e0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sttxg5n27hh5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95422086-7eb5-4fab-930f-f78b5c9185bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95422086-7eb5-4fab-930f-f78b5c9185bc\",\n      \"name\": \"95422086-7eb5-4fab-930f-f78b5c9185bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T00:05:50.759671414Z\",\n      \"endTime\": \"2026-10-01T00:24:59.018405341Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sttxg5n27hh5--managed/providers/Microsoft.Compute/virtualMachines/m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sttxg5n27hh5--managed/providers/Microsoft.Compute/virtualMachines/m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"22a9d6b1accb3a7a80bdce02cca313fe7effb40420d665604f27adb43e6bf9fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"5d856dedaa58bb923d9cef2471b19783816a8d29d3cf957e7044257ee539a42c","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d541a0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-mv9rpr\" in resourcegroup=\"rg-np-skip-minor-mv9rpr-lccf5r2wcvcf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-mv9rpr-lccf5r2wcvcf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-mv9rpr/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d541a0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-mv9rpr\" in resourcegroup=\"rg-np-skip-minor-mv9rpr-lccf5r2wcvcf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-mv9rpr-lccf5r2wcvcf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-mv9rpr/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4ca8c0495419322ec8b177919aac6b52c05361c73321dc31fdf98d2b8c559195","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"733cfb2369652cdeae179de94ce49731d6a1d3599568a270c3308ae212724dfa","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-5-0-b96xbr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e76f48\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-5-0-b96xbr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e76f48\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"5c190c083d2b9cb1ee778d86aa7e70090c86473421e1417e9ac0d5a28f2a9385","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"deca178dbaa2cc480c8418c70ba18703fd89f0904bca91f4e92e8446fbaf30cc","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b8bc0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-sgrs2v\" in resourcegroup=\"rg-np-version-upgrade-sgrs2v-l46fqcfczcg5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-sgrs2v-l46fqcfczcg5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-sgrs2v/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b8bc0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-sgrs2v\" in resourcegroup=\"rg-np-version-upgrade-sgrs2v-l46fqcfczcg5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-sgrs2v-l46fqcfczcg5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-sgrs2v/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6e8c421e1d078a7bbf6d3f39524024c515be649f120e55c98c9514c4428f5130","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"a4aa188ac160ec2ff4c13c20c75a17504feb16474f727e3f622fd64eae9523cf","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:178]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001069ba8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-29-171841, targetMinor=5.1.0-0.nightly-multi-2026-09-30-105558) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-30-105558\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:178]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001069ba8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-29-171841, targetMinor=5.1.0-0.nightly-multi-2026-09-30-105558) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-30-105558\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"7ffcd43314133b91ce526d2b7c99d89e2700f9caf601291f10b4323657545f06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"2abb3bce100e636041d87fa3d6c3423531ab2ba3b2d69922e70f8a53d50142a2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to verify HCP cluster viability for patch-tags test\nUnexpected error:\n    \u003c*errors.joinError | 0xc001619020\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to verify HCP cluster viability for patch-tags test\nUnexpected error:\n    \u003c*errors.joinError | 0xc001619020\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"dfb6d238afc40021735949f38ab91051e6415c246a90fbaad9992adb2ee67f46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"9b73350e92406053dbb562adaf95183f694e1699162837a7daf5336133bff3ca","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001472300\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4zkcv5\" resourcegroup=\"rg-np-dg-minor-4zkcv5-pjsq2w8jb2dl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4zkcv5-pjsq2w8jb2dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4zkcv5/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001472300\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4zkcv5\" resourcegroup=\"rg-np-dg-minor-4zkcv5-pjsq2w8jb2dl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4zkcv5-pjsq2w8jb2dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4zkcv5/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e031728508f3ca6ba83242f00f52bc9d1dab23953d0e6987584352a8630a6e67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"6313ae0584d2d0c419cdb14a4f57c8d3049e608c9cfe03eac9cc76d52450b807","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fzw8hf\" for cluster \"cluster-nightly-4-20-fzw8hf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca4c80\u003e: \n    failed to create NodePool np-fzw8hf: failed starting nodepool creation \"np-fzw8hf\" for cluster \"cluster-nightly-4-20-fzw8hf\" in resourcegroup=\"rg-nightly-4-20-tfqsvtwn4ns7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-tfqsvtwn4ns7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-fzw8hf/nodePools/np-fzw8hf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fzw8hf\" for cluster \"cluster-nightly-4-20-fzw8hf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca4c80\u003e: \n    failed to create NodePool np-fzw8hf: failed starting nodepool creation \"np-fzw8hf\" for cluster \"cluster-nightly-4-20-fzw8hf\" in resourcegroup=\"rg-nightly-4-20-tfqsvtwn4ns7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-tfqsvtwn4ns7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-fzw8hf/nodePools/np-fzw8hf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105409887751114752","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-30T21:30:08Z","started_at":"2026-09-30T21:30:08Z","completed_at":"2026-09-30T23:59:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:04:58.992493806Z","timing_metadata_checked_at":"2026-10-01T00:04:58.992493806Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:04:58.995646296Z","region_metadata_checked_at":"2026-10-01T00:04:58.995646296Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105115529034338304","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T02:00:27Z","started_at":"2026-09-30T02:00:27Z","completed_at":"2026-09-30T04:28:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:45:46.278018404Z","timing_metadata_checked_at":"2026-09-30T04:45:46.278018404Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T04:45:46.283201489Z","region_metadata_checked_at":"2026-09-30T04:45:46.283201489Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a8d7b06791ceae1efd644cc2fa8bffa546718fecee381183b5d1b67ec88a6845","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105115529034338304","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"2f89faaaf7b43a2918fd4eba43a3db8eedeb08cfc96d4a5da653b1792d8738d4","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c85c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-wbpxbvspwkdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/44b3f004-e16f-436f-bdc7-4652956b17b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/44b3f004-e16f-436f-bdc7-4652956b17b0\",\n      \"name\": \"44b3f004-e16f-436f-bdc7-4652956b17b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:12:50.954793928Z\",\n      \"endTime\": \"2026-09-30T02:31:50.125456331Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-mhjdk-jdk87: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wbpxbvspwkdl--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mhjdk-jdk87 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-mhjdk-jdk87: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wbpxbvspwkdl--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mhjdk-jdk87 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c85c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-wbpxbvspwkdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/44b3f004-e16f-436f-bdc7-4652956b17b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/44b3f004-e16f-436f-bdc7-4652956b17b0\",\n      \"name\": \"44b3f004-e16f-436f-bdc7-4652956b17b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:12:50.954793928Z\",\n      \"endTime\": \"2026-09-30T02:31:50.125456331Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-mhjdk-jdk87: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wbpxbvspwkdl--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mhjdk-jdk87 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-mhjdk-jdk87: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wbpxbvspwkdl--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mhjdk-jdk87 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fa466adb5b5a808bb74b6c513da078593becc3704c6862ab38fbee4c6833d80a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105115529034338304","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"342894fb948ca9c214cc8f0f4cb97e686763db2288c64ee0b0f2635fc366a42d","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001415050\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001415050\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":5,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T23:00:28Z","started_at":"2026-09-30T23:00:28Z","completed_at":"2026-10-01T00:34:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:36:09.415562976Z","timing_metadata_checked_at":"2026-10-01T00:36:09.415562976Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:36:09.418072268Z","region_metadata_checked_at":"2026-10-01T00:36:09.418072268Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"250567d2dad0ec032f94249d9bcf7d29c31828244fc119b5b97a3a0fccce9c9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"297ae9530982a4810dfc39b4017f6105e35c27ea06de54d22f389e155e41a619","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-srbb2hbvsczt/cluster-nightly-5-1-vxb9dm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1af40\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-vxb9dm: failed waiting for cluster=\"cluster-nightly-5-1-vxb9dm\" in resourcegroup=\"rg-nightly-5-1-srbb2hbvsczt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"name\": \"fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:26:50.189510673Z\",\n      \"endTime\": \"2026-09-30T23:55:57.188151115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-105558 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-srbb2hbvsczt/cluster-nightly-5-1-vxb9dm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1af40\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-vxb9dm: failed waiting for cluster=\"cluster-nightly-5-1-vxb9dm\" in resourcegroup=\"rg-nightly-5-1-srbb2hbvsczt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"name\": \"fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:26:50.189510673Z\",\n      \"endTime\": \"2026-09-30T23:55:57.188151115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-105558 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bdaff9adc205561255863f7f3862c4a4a3cdfd05505c7e10e9a53b9e2959cba5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4a8ea4ca61d811f2d283cae7586304bb87d7a7bfe6bf5fc3e9a82ebe6229301e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"754bee22198c76ca5ecb7193f9b82365056113e3559f87b518bbbd3678c80bd6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014907c0\u003e: \n    failed to create NodePool np-prg87r: failed starting nodepool creation \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\" in resourcegroup=\"rg-nightly-4-20-kjrkpmc2vcnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kjrkpmc2vcnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-prg87r/nodePools/np-prg87r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014907c0\u003e: \n    failed to create NodePool np-prg87r: failed starting nodepool creation \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\" in resourcegroup=\"rg-nightly-4-20-kjrkpmc2vcnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kjrkpmc2vcnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-prg87r/nodePools/np-prg87r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7570604bafa6565b58803f786890e718248e56e844f7447e4608f8464000e8f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"08174c295f58ea65eae7d6210ad569d41e3b1cdc0b0a84a4e29c556f000b91f6","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00154c140\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-49bq5x\" resourcegroup=\"rg-np-dg-minor-49bq5x-7f6vt9z24c7x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-49bq5x-7f6vt9z24c7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-49bq5x/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00154c140\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-49bq5x\" resourcegroup=\"rg-np-dg-minor-49bq5x-7f6vt9z24c7x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-49bq5x-7f6vt9z24c7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-49bq5x/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8b5dbf275e1e89f385dbef51b011d8fb20efbc788ea7bbd6fead6581e08ab58d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"a9ee924e57f9ab9bbddd25402eaa20f29c4967eedecb92e4e71d8d0c06ef84c2","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b92100\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-qglcmw\" in resourcegroup=\"rg-np-version-upgrade-qglcmw-2xxcc62c56zb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-qglcmw-2xxcc62c56zb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-qglcmw/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b92100\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-qglcmw\" in resourcegroup=\"rg-np-version-upgrade-qglcmw-2xxcc62c56zb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-qglcmw-2xxcc62c56zb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-qglcmw/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a63905cc2169d208d54429ce41641ab7ea80696a3d8afea808e54ff74867de7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"b7ec09633e70cfe3b7246a6bc21475b497dc8e51890bc8b252d77a2143ce919b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d74240\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-qlzv6l\" in resourcegroup=\"rg-np-skip-minor-qlzv6l-d9kp5xhmf6nr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-qlzv6l-d9kp5xhmf6nr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-qlzv6l/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d74240\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-qlzv6l\" in resourcegroup=\"rg-np-skip-minor-qlzv6l-d9kp5xhmf6nr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-qlzv6l-d9kp5xhmf6nr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-qlzv6l/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","bdaff9adc205561255863f7f3862c4a4a3cdfd05505c7e10e9a53b9e2959cba5"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T15:51:35Z","started_at":"2026-09-30T15:51:35Z","completed_at":"2026-09-30T18:08:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:29:07.732164822Z","timing_metadata_checked_at":"2026-09-30T18:29:07.732164822Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:29:07.734945814Z","region_metadata_checked_at":"2026-09-30T18:29:07.734945814Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1f75451f0f3fa7c6b2d4b31a00b4ff21302c48497c25f2fe7aebb0575fd2b1fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"3933609f5fce2e295de9ab5e9c2e68be690e233f3f4534aede4c689e2581a030","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000119c40\u003e: \n    failed to create NodePool np-djrg27: failed waiting for nodepool=\"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\" in resourcegroup=\"rg-candidate-4-20-hwlm5v4jsr9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"name\": \"94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:46:47.162303776Z\",\n      \"endTime\": \"2026-09-30T17:05:53.284012473Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000119c40\u003e: \n    failed to create NodePool np-djrg27: failed waiting for nodepool=\"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\" in resourcegroup=\"rg-candidate-4-20-hwlm5v4jsr9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"name\": \"94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:46:47.162303776Z\",\n      \"endTime\": \"2026-09-30T17:05:53.284012473Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8ab889d7329c55632f3e73c5be3ffc486e2c2589f066b8767007ab496d59c6a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"a26abfcbde513ec1ccfb520656c877d6a0b6d347a97e0f7cca974af4807b4a2b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001703c60\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-jmc9c7\" in resourcegroup=\"rg-np-skip-minor-jmc9c7-9dtxqp29vbsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\",\n      \"name\": \"d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:21:41.15243756Z\",\n      \"endTime\": \"2026-09-30T17:40:44.044506032Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jmc9c7-9dtxqp29vbsr-np-sm-jmc9c7--managed/providers/Microsoft.Compute/virtualMachines/c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jmc9c7-9dtxqp29vbsr-np-sm-jmc9c7--managed/providers/Microsoft.Compute/virtualMachines/c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001703c60\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-jmc9c7\" in resourcegroup=\"rg-np-skip-minor-jmc9c7-9dtxqp29vbsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\",\n      \"name\": \"d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:21:41.15243756Z\",\n      \"endTime\": \"2026-09-30T17:40:44.044506032Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jmc9c7-9dtxqp29vbsr-np-sm-jmc9c7--managed/providers/Microsoft.Compute/virtualMachines/c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jmc9c7-9dtxqp29vbsr-np-sm-jmc9c7--managed/providers/Microsoft.Compute/virtualMachines/c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dfb27f0ee5b3afdbf71aab9103209d7b0550b3614d5bb16625715b0f4d6b3cb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"2de55e74f48dd0f13326c20bf872d192fa878f413d1746046060d6f653450349","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001116780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-vh7hggp8cw8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\",\n      \"name\": \"eed44f57-1a91-4587-8901-935588d355ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:09:44.769920829Z\",\n      \"endTime\": \"2026-09-30T17:28:44.009160044Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001116780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-vh7hggp8cw8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\",\n      \"name\": \"eed44f57-1a91-4587-8901-935588d355ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:09:44.769920829Z\",\n      \"endTime\": \"2026-09-30T17:28:44.009160044Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management"}},{"row_id":"ee2506ef305781ed5e9ed673355e6af467a7570e9cd2823743c49f2793bcc57b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"4aed755d2532b13cc80a0d2fcd018700a14d4af99ab0fa4f4dfa6f963040d21e","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015c0ae0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-twtdf7gdskgb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"name\": \"ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:25:28.987424124Z\",\n      \"endTime\": \"2026-09-30T16:44:30.182558055Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015c0ae0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-twtdf7gdskgb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"name\": \"ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:25:28.987424124Z\",\n      \"endTime\": \"2026-09-30T16:44:30.182558055Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fa7353b081c974760137d2173ac430acdca29ab370ea1f5df1e57b6bcd747605","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"213150af94f11ecc8fc032938950c97f98aae10d4ca8c4c3d158d38b210c0554","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c2500\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-xhfvnf\" in resourcegroup=\"rg-np-noedge-xhfvnf-d6cdpdqmxzz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"name\": \"79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:42.49451817Z\",\n      \"endTime\": \"2026-09-30T17:39:43.993225307Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c2500\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-xhfvnf\" in resourcegroup=\"rg-np-noedge-xhfvnf-d6cdpdqmxzz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"name\": \"79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:42.49451817Z\",\n      \"endTime\": \"2026-09-30T17:39:43.993225307Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T12:10:26Z","started_at":"2026-09-30T12:10:26Z","completed_at":"2026-09-30T14:28:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:48:05.156360818Z","timing_metadata_checked_at":"2026-09-30T14:48:05.156360818Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:48:05.163389396Z","region_metadata_checked_at":"2026-09-30T14:48:05.163389396Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"5631922113930af9bf7334768a4ca83318a4bb52715230b6644d21b5d83e740e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","failed_at":"e2e","signature_id":"06f24328299a032f056108312696270e080fc953f45840834663f37907dc3db6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00148e9c0\u003e: \n    resource group \"rg-zstream-upgrade-5-0-l6m7gzggxh5z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-mrptx7\" in resourcegroup=\"rg-zstream-upgrade-5-0-l6m7gzggxh5z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"name\": \"d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:28:47.89996524Z\",\n      \"endTime\": \"2026-09-30T13:52:47.080528249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5bj2pp5rm42moqm553gsgpuq5afnl4 still exists (deletion dispatched at 2026-09-30T13:28:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00148e9c0\u003e: \n    resource group \"rg-zstream-upgrade-5-0-l6m7gzggxh5z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-mrptx7\" in resourcegroup=\"rg-zstream-upgrade-5-0-l6m7gzggxh5z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"name\": \"d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:28:47.89996524Z\",\n      \"endTime\": \"2026-09-30T13:52:47.080528249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5bj2pp5rm42moqm553gsgpuq5afnl4 still exists (deletion dispatched at 2026-09-30T13:28:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8e372e8cd22844b25567b8936aae520315cefe944c8204b6d4b9f58401136e31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","failed_at":"e2e","signature_id":"ba33ade4a966c1de67f013ea1140c2c38fb60d9c849af3a44a8bb8f1d3d3e42b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ed920\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-rkbcqh\" in resourcegroup=\"rg-np-skip-minor-rkbcqh-h7clvkbvfrzv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\",\n      \"name\": \"0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:38:48.671242236Z\",\n      \"endTime\": \"2026-09-30T13:57:57.057793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-rkbcqh-h7clvkbvfrzv-np-sm-rkbcqh--managed/providers/Microsoft.Compute/virtualMachines/i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-rkbcqh-h7clvkbvfrzv-np-sm-rkbcqh--managed/providers/Microsoft.Compute/virtualMachines/i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ed920\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-rkbcqh\" in resourcegroup=\"rg-np-skip-minor-rkbcqh-h7clvkbvfrzv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\",\n      \"name\": \"0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:38:48.671242236Z\",\n      \"endTime\": \"2026-09-30T13:57:57.057793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-rkbcqh-h7clvkbvfrzv-np-sm-rkbcqh--managed/providers/Microsoft.Compute/virtualMachines/i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-rkbcqh-h7clvkbvfrzv-np-sm-rkbcqh--managed/providers/Microsoft.Compute/virtualMachines/i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"91698a44aad5ca32c424df508296f3f679852764e7f79c96c1e59cdf00596e29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","failed_at":"e2e","signature_id":"cda87c02afe385de5c6e530de6be078040d84e3fa8634ad1d769024158056676","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c62580\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4ldsbj98fhm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b038c9-d01f-4cce-8da4-821ff1fad693\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b038c9-d01f-4cce-8da4-821ff1fad693\",\n      \"name\": \"d5b038c9-d01f-4cce-8da4-821ff1fad693\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:41:58.878961581Z\",\n      \"endTime\": \"2026-09-30T13:01:07.023295945Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4ldsbj98fhm7--managed/providers/Microsoft.Compute/virtualMachines/p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4ldsbj98fhm7--managed/providers/Microsoft.Compute/virtualMachines/p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c62580\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4ldsbj98fhm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b038c9-d01f-4cce-8da4-821ff1fad693\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b038c9-d01f-4cce-8da4-821ff1fad693\",\n      \"name\": \"d5b038c9-d01f-4cce-8da4-821ff1fad693\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:41:58.878961581Z\",\n      \"endTime\": \"2026-09-30T13:01:07.023295945Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4ldsbj98fhm7--managed/providers/Microsoft.Compute/virtualMachines/p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4ldsbj98fhm7--managed/providers/Microsoft.Compute/virtualMachines/p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"939f8c5af8420a6aa6e291389f9e8135d165bea7a857a4cd7eb1733e587fdcc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","failed_at":"e2e","signature_id":"b859706588c59cf13ba01690598c1e296b21c66674acdd2387e5df98da83bb15","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd79a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd79a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83447d3d19023b63c00022eba710d55e1a0090317aba9cb3e48328dc0b521261","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","83447d3d19023b63c00022eba710d55e1a0090317aba9cb3e48328dc0b521261","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105187694681264128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T06:47:13Z","started_at":"2026-09-30T06:47:13Z","completed_at":"2026-09-30T09:01:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:11:41.469641612Z","timing_metadata_checked_at":"2026-09-30T09:11:41.469641612Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:11:41.472457204Z","region_metadata_checked_at":"2026-09-30T09:11:41.472457204Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4bf1d48fc5aa1eeeed8e1e4cef061704ad02a02b051326fdfafa07cb35cf2eb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105187694681264128","occurred_at":"2026-09-30T06:47:13Z","failed_at":"e2e","signature_id":"e3db6908a51e323a629ef61d2f3ca60b3da46a872c365d1cca8d13ce3880374f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a8a4b0\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-b6jrz8\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"name\": \"8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:07:19.758629115Z\",\n      \"endTime\": \"2026-09-30T08:31:25.87901072Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56tcneq332c0ht4fv2g22jot5kdluc still exists (deletion dispatched at 2026-09-30T08:07:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a8a4b0\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-b6jrz8\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"name\": \"8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:07:19.758629115Z\",\n      \"endTime\": \"2026-09-30T08:31:25.87901072Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56tcneq332c0ht4fv2g22jot5kdluc still exists (deletion dispatched at 2026-09-30T08:07:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b1d3cd1631f041b5259319fed4389bb786eac41289d1c10810e2c1ed7d395fae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105187694681264128","occurred_at":"2026-09-30T06:47:13Z","failed_at":"e2e","signature_id":"793343d9f2198c07b4a1defe40164b7665fed5f9625606d1878699fdfd9f02db","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0013950a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"name\": \"626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:05:30.277326796Z\",\n      \"endTime\": \"2026-09-30T08:24:30.136180053Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0013950a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"name\": \"626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:05:30.277326796Z\",\n      \"endTime\": \"2026-09-30T08:24:30.136180053Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T02:00:27Z","started_at":"2026-09-30T02:00:27Z","completed_at":"2026-09-30T03:58:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:15:05.778248446Z","timing_metadata_checked_at":"2026-09-30T04:15:05.778248446Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T04:15:05.781206637Z","region_metadata_checked_at":"2026-09-30T04:15:05.781206637Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"137960408ac2fe9cc2e1e0f2be91572b0d3eda1c1c26a596515cd4481dac7a09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"5e2362cdd385e535c236eb25d2af3bedec9c45598cf42305119a1c2199da01c0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lssckk\" for cluster \"cluster-candidate-4-22-lssckk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0d380\u003e: \n    failed to create NodePool np-lssckk: failed waiting for nodepool=\"np-lssckk\" for cluster \"cluster-candidate-4-22-lssckk\" in resourcegroup=\"rg-candidate-4-22-bqwkx6ssnq7c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b6149906-5b1c-417e-9e9c-3b2c8a352da8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b6149906-5b1c-417e-9e9c-3b2c8a352da8\",\n      \"name\": \"b6149906-5b1c-417e-9e9c-3b2c8a352da8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:38:56.070433953Z\",\n      \"endTime\": \"2026-09-30T02:57:58.4595049Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-bqwkx6ssnq7c-candidate-lssckk--managed/providers/Microsoft.Compute/virtualMachines/e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-bqwkx6ssnq7c-candidate-lssckk--managed/providers/Microsoft.Compute/virtualMachines/e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lssckk\" for cluster \"cluster-candidate-4-22-lssckk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0d380\u003e: \n    failed to create NodePool np-lssckk: failed waiting for nodepool=\"np-lssckk\" for cluster \"cluster-candidate-4-22-lssckk\" in resourcegroup=\"rg-candidate-4-22-bqwkx6ssnq7c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b6149906-5b1c-417e-9e9c-3b2c8a352da8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b6149906-5b1c-417e-9e9c-3b2c8a352da8\",\n      \"name\": \"b6149906-5b1c-417e-9e9c-3b2c8a352da8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:38:56.070433953Z\",\n      \"endTime\": \"2026-09-30T02:57:58.4595049Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-bqwkx6ssnq7c-candidate-lssckk--managed/providers/Microsoft.Compute/virtualMachines/e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-bqwkx6ssnq7c-candidate-lssckk--managed/providers/Microsoft.Compute/virtualMachines/e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"287863c149517e9259d41fd5ddb52b1ac7fe47bd912cd1a7f835afcf21754c9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"d88867851ef4d43076446e92a3c85889999fd11a09b6284afc83108a89d24f58","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001035640\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4kv8jbqzgm6h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0011df08-d47e-405c-b953-36672f501c27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0011df08-d47e-405c-b953-36672f501c27\",\n      \"name\": \"0011df08-d47e-405c-b953-36672f501c27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T03:17:50.983140083Z\",\n      \"endTime\": \"2026-09-30T03:36:56.457139595Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4kv8jbqzgm6h--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4kv8jbqzgm6h--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001035640\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4kv8jbqzgm6h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0011df08-d47e-405c-b953-36672f501c27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0011df08-d47e-405c-b953-36672f501c27\",\n      \"name\": \"0011df08-d47e-405c-b953-36672f501c27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T03:17:50.983140083Z\",\n      \"endTime\": \"2026-09-30T03:36:56.457139595Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4kv8jbqzgm6h--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4kv8jbqzgm6h--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c029d77569b7cc39a439f01d39feca6655374b07db10ca40bcc38570ac2412e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"f280960e298b057c6eef0496e65821ec4103a9ef2176aedafe051906baf460a7","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008827a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-tb59h9fbmsnp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69cf1894-8af7-4f4d-985a-7d39b5ffb985\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69cf1894-8af7-4f4d-985a-7d39b5ffb985\",\n      \"name\": \"69cf1894-8af7-4f4d-985a-7d39b5ffb985\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:15:06.409498278Z\",\n      \"endTime\": \"2026-09-30T02:34:10.118150935Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tb59h9fbmsnp--managed/providers/Microsoft.Compute/virtualMachines/o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tb59h9fbmsnp--managed/providers/Microsoft.Compute/virtualMachines/o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008827a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-tb59h9fbmsnp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69cf1894-8af7-4f4d-985a-7d39b5ffb985\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69cf1894-8af7-4f4d-985a-7d39b5ffb985\",\n      \"name\": \"69cf1894-8af7-4f4d-985a-7d39b5ffb985\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:15:06.409498278Z\",\n      \"endTime\": \"2026-09-30T02:34:10.118150935Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tb59h9fbmsnp--managed/providers/Microsoft.Compute/virtualMachines/o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tb59h9fbmsnp--managed/providers/Microsoft.Compute/virtualMachines/o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dafb1ae9739417d29a65d4fcca8a79d4531b8e7c78aaa5d616c471462c06b6f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"959da02202a889de6291751a794c7fb5d01b45f09a2d70f1fa8fe082a480a903","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016d54b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016d54b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"dd046f4f0751e6475d45e665396196320d5fe6e336b8818055e647f96f904f2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"d4afdcb13b7abb84b80ed486bbab0e753e39b3cf0759b719778c721a28c9e54b","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c25d80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-jtns6kb9n72s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13d22eca-d7af-43b2-9ab7-e630c81c0521\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13d22eca-d7af-43b2-9ab7-e630c81c0521\",\n      \"name\": \"13d22eca-d7af-43b2-9ab7-e630c81c0521\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:16:05.748001285Z\",\n      \"endTime\": \"2026-09-30T02:35:10.119119459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jtns6kb9n72s--managed/providers/Microsoft.Compute/virtualMachines/q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jtns6kb9n72s--managed/providers/Microsoft.Compute/virtualMachines/q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c25d80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-jtns6kb9n72s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13d22eca-d7af-43b2-9ab7-e630c81c0521\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13d22eca-d7af-43b2-9ab7-e630c81c0521\",\n      \"name\": \"13d22eca-d7af-43b2-9ab7-e630c81c0521\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:16:05.748001285Z\",\n      \"endTime\": \"2026-09-30T02:35:10.119119459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jtns6kb9n72s--managed/providers/Microsoft.Compute/virtualMachines/q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jtns6kb9n72s--managed/providers/Microsoft.Compute/virtualMachines/q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}