{"meta":{"date":"2026-10-09","timezone":"UTC","generated_at":"2026-10-14T05:13:39Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":59,"failed_runs":42,"runs_with_occurrences":42,"runs_with_failure_pattern_matches":38,"runs_with_unmatched_occurrences":4,"runs_without_occurrence_rows":17,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108698034241015808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"open","pr_sha":"9df3071737b6aa6b7a196febdabb2b01ee6b0499","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T23:16:03Z","started_at":"2026-10-09T23:16:03Z","completed_at":"2026-10-10T01:27:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T01:42:24.284458516Z","timing_metadata_checked_at":"2026-10-10T01:42:24.284458516Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T01:42:24.288385804Z","region_metadata_checked_at":"2026-10-10T01:42:24.288385804Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108694594228064256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"closed","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T23:02:23Z","started_at":"2026-10-09T23:02:23Z","completed_at":"2026-10-10T01:20:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T01:31:38.672458754Z","timing_metadata_checked_at":"2026-10-10T01:31:38.672458754Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T01:31:38.67667614Z","region_metadata_checked_at":"2026-10-10T01:31:38.67667614Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a878d18202891fb8dad49b2d67cb7c94f0bc0c4c058be754fb4317cb47c20d90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108694594228064256","occurred_at":"2026-10-09T23:02:23Z","failed_at":"alert","signature_id":"45c56709fb0815fc5125e96d111279ae3d1aed3d93168dc727565ba057fd9a0f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-10T00:27:09Z\n Ended: 2026-10-10T00:33:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8064256-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbj1c9chavhk90qkot47r4fc981jv3j-p7l5u8c0n3u2s1c\", pod=\"router-5869c468cd-cr9j5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbj1c9chavhk90qkot47r4fc981jv3j-p7l5u8c0n3u2s1c/router-5869c468cd-cr9j5 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108694589719187456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7227,"pr_state":"closed","pr_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","final_merged_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T23:02:22Z","started_at":"2026-10-09T23:02:22Z","completed_at":"2026-10-10T01:25:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T01:42:24.262058185Z","timing_metadata_checked_at":"2026-10-10T01:42:24.262058185Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T01:42:24.265836573Z","region_metadata_checked_at":"2026-10-10T01:42:24.265836573Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"13174d9f70bef03f3ee6c538e6afafad28c868678a9bcad894654d0dc37396e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108694589719187456","occurred_at":"2026-10-09T23:02:22Z","failed_at":"alert","signature_id":"ce44dc83babfef2811856faa5dcaa4ffd6178cd17bba577f8a5125d2ff96ccb0","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-10T00:52:27Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9187456-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.0:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9187456/providers/microsoft.monitor/accounts/services-j9187456\", namespace=\"klusterlet-2tbj2sv7sdukmvi25teg7m48qmbbnkqf\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-ns4ml\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tbj2sv7sdukmvi25teg7m48qmbbnkqf on cluster ci01-j9187456-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7450/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108694431543595008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7450,"pr_state":"closed","pr_sha":"e5a15715dba2cbe9a936670e8fab767d88314f32","final_merged_sha":"e5a15715dba2cbe9a936670e8fab767d88314f32","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T23:01:44Z","started_at":"2026-10-09T23:01:44Z","completed_at":"2026-10-10T01:15:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T01:31:38.800771049Z","timing_metadata_checked_at":"2026-10-10T01:31:38.800771049Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T01:31:38.803777839Z","region_metadata_checked_at":"2026-10-10T01:31:38.803777839Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108694038969323520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"open","pr_sha":"d5f9707574b74ff42aabca7b2c533579503fd071","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T23:00:11Z","started_at":"2026-10-09T23:00:11Z","completed_at":"2026-10-10T01:34:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T01:42:24.447428414Z","timing_metadata_checked_at":"2026-10-10T01:42:24.447428414Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T01:42:24.450186105Z","region_metadata_checked_at":"2026-10-10T01:42:24.450186105Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7448/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108692920105177088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7448,"pr_state":"open","pr_sha":"b878a2d1a66efeedf8247c57b747d18b398e8814","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T22:55:44Z","started_at":"2026-10-09T22:55:44Z","completed_at":"2026-10-10T01:35:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T01:52:57.489004681Z","timing_metadata_checked_at":"2026-10-10T01:52:57.489004681Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T01:52:57.49257367Z","region_metadata_checked_at":"2026-10-10T01:52:57.49257367Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"8932a9dc3c8d2e48151172a323fdc1065f2d715a6cdfdcf561580c443e05db03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7448/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108692920105177088","occurred_at":"2026-10-09T22:55:44Z","failed_at":"alert","signature_id":"28fcb1c27858e1387aacc2dacf213e3c55e8e38d44b6bf8ef602a149e90b5993","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-10T01:13:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5177088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.148:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5177088/providers/microsoft.monitor/accounts/hcps-j5177088\", namespace=\"ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k\", pod=\"cloud-network-config-controller-7847666f8b-xtgjl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"78775d9a-b0ca-4c4d-bd49-d00b20e49e68\"\n Description: Pod ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k/cloud-network-config-controller-7847666f8b-xtgjl (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-10T01:13:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5177088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.148:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5177088/providers/microsoft.monitor/accounts/hcps-j5177088\", namespace=\"ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k\", pod=\"csi-snapshot-controller-669f7dfb7b-68ftn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6585df9e-c94d-48b0-a688-96a3851ac4dd\"\n Description: Pod ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k/csi-snapshot-controller-669f7dfb7b-68ftn (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-10T01:14:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5177088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.148:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5177088/providers/microsoft.monitor/accounts/hcps-j5177088\", namespace=\"ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k\", pod=\"csi-snapshot-controller-669f7dfb7b-68ftn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6585df9e-c94d-48b0-a688-96a3851ac4dd\"\n Description: Pod ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k/csi-snapshot-controller-669f7dfb7b-68ftn (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-10T01:14:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5177088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.148:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5177088/providers/microsoft.monitor/accounts/hcps-j5177088\", namespace=\"ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k\", pod=\"cloud-network-config-controller-7847666f8b-xtgjl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"78775d9a-b0ca-4c4d-bd49-d00b20e49e68\"\n Description: Pod ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k/cloud-network-config-controller-7847666f8b-xtgjl (controller) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"df6d19f258e0bab0ed37dcadaeff8327cd8508fecb9ad7aa4ce6fd1dd50d9f6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7448/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108692920105177088","occurred_at":"2026-10-09T22:55:44Z","failed_at":"alert","signature_id":"7de35d5024d3b3010d077617b0ac7fd8f62bcc84b4d231065a5dc6177210187d","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-10T00:42:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5177088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.148:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5177088/providers/microsoft.monitor/accounts/hcps-j5177088\", namespace=\"ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mchpr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-10T00:43:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5177088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.148:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5177088/providers/microsoft.monitor/accounts/hcps-j5177088\", namespace=\"ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mchpr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbitkkae9upoufmp2r1juh69i34b35n-y0t1t6j2p7i5j5k/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108691657791639552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7449,"pr_state":"closed","pr_sha":"d11cff8855176f35005a5b3f38b46f917250a680","final_merged_sha":"d11cff8855176f35005a5b3f38b46f917250a680","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T22:50:35Z","started_at":"2026-10-09T22:50:35Z","completed_at":"2026-10-09T23:31:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T23:48:42.576840976Z","timing_metadata_checked_at":"2026-10-09T23:48:42.576840976Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-09T23:48:42.579459568Z","region_metadata_checked_at":"2026-10-10T00:49:45.569175041Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e80472a92bd3b6e539c556ded80dd47b9b9614d43497bfef670e185bda84e54f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7449/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108691657791639552","occurred_at":"2026-10-09T22:50:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108685090446905344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"open","pr_sha":"bca9cb7a9dff975def2a7e09a9485d16e05c85aa","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T22:24:37Z","started_at":"2026-10-09T22:24:37Z","completed_at":"2026-10-09T22:43:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T22:56:22.509405274Z","timing_metadata_checked_at":"2026-10-09T22:56:22.509405274Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-09T22:56:22.513173263Z","region_metadata_checked_at":"2026-10-09T23:58:44.750748698Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"784e3b5077252d6d4e309297bbc30d0bba1310e428f618fa82dc31649b403181","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108685090446905344","occurred_at":"2026-10-09T22:24:37Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108676539800358912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7447,"pr_state":"closed","pr_sha":"1245d66a84eb05f562bb3efd6fedfd81508ee0d3","final_merged_sha":"1245d66a84eb05f562bb3efd6fedfd81508ee0d3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T21:50:39Z","started_at":"2026-10-09T21:50:39Z","completed_at":"2026-10-10T00:08:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T00:19:11.398568311Z","timing_metadata_checked_at":"2026-10-10T00:19:11.398568311Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T00:19:11.401482203Z","region_metadata_checked_at":"2026-10-10T00:19:11.401482203Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108676067857272832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"closed","pr_sha":"96338d0d4c9fe0829812f6efeebdd1fea8adc217","final_merged_sha":"96338d0d4c9fe0829812f6efeebdd1fea8adc217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T21:48:46Z","started_at":"2026-10-09T21:48:46Z","completed_at":"2026-10-10T00:05:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T00:19:11.25779749Z","timing_metadata_checked_at":"2026-10-10T00:19:11.25779749Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T00:19:11.261294381Z","region_metadata_checked_at":"2026-10-10T00:19:11.261294381Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"048d66fb13d18af719f153888dded775cf40029e8ddbc7904ab6a3394cce0cc8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108676067857272832","occurred_at":"2026-10-09T21:48:46Z","failed_at":"e2e","signature_id":"133758dc096cedae43ff2559d3a74348d74fa3017fff8b919bd6d528e1c03295","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3475813fe160\u003e: \n DNS for route host agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3475813fe0e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3475813fe160\u003e: \n DNS for route host agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3475813fe0e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p6pqz.apps.aro.v20261001.e223.j7272832.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108674180797960192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7249,"pr_state":"closed","pr_sha":"4d2db88e2c3c4e0894b79d6ffabc5a6460dc5ff6","final_merged_sha":"4d2db88e2c3c4e0894b79d6ffabc5a6460dc5ff6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T21:41:16Z","started_at":"2026-10-09T21:41:16Z","completed_at":"2026-10-09T22:12:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T22:24:38.386041498Z","timing_metadata_checked_at":"2026-10-09T22:24:38.386041498Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T22:24:38.389744087Z","region_metadata_checked_at":"2026-10-09T22:24:38.389744087Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ad877550f38e93d25470011d17bbbf00996a444713e7f9b7cd525012d5285abf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108674180797960192","occurred_at":"2026-10-09T21:41:16Z","failed_at":"provision","signature_id":"abe4c5c2b231f43f69eb7923fecd7d0c54d26cc4bcd5d781e5832765ceb537cc","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-09T22:05:07.216Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-09T22:06:09.182Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-09T22:05:07.225651655Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-09T22:05:08.960792766Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-09T22:05:09.292773856Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-09T22:06:09.178268762Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"collecting deployment operations\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"err\\\":{\\\"errorCode\\\":\\\"DeploymentNotFound\\\",\\\"statusCode\\\":404,\\\"errorMessage\\\":\\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7960192-mgmt-2/deployments/aks-cluster-create/operations\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE: DeploymentNotFound\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentNotFound\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Deployment 'aks-cluster-create' could not be found.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"}}\\n{\\\"time\\\":\\\"2026-10-09T22:06:09.178359704Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7960192-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7960192-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"submitting deployment aks-cluster-create: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7960192-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 400: 400 Bad Request\\\\nERROR CODE: InvalidTemplateDeployment\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'aks-cluster-create' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.ContainerService/managedClusters (2026-04-02-preview)' reported preflight validation errors. Tracking id is 'af747446-bfb7-43e9-a875-3f8953ec8e67'. See inner errors for details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ListResourceSKUsFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Preflight validation check for resource(s) for container service ci01-j7960192-mgmt-2 in resource group hcp-underlay-ci01-j7960192-mgmt-2 failed. Message: List resource SKUs in subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX failed. Details: \\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"1m1.952725139s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64bb2607db9a31af645f91fcf6ffbdb70d8bca63638eea03e861ce2387c49227","failure_pattern":"ERROR CODE: DeploymentNotFound; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["64bb2607db9a31af645f91fcf6ffbdb70d8bca63638eea03e861ce2387c49227"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108673315018117120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7446,"pr_state":"closed","pr_sha":"43d0ff6c1035d02609f6502b8fdd1292ebf84afa","final_merged_sha":"43d0ff6c1035d02609f6502b8fdd1292ebf84afa","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T21:37:50Z","started_at":"2026-10-09T21:37:50Z","completed_at":"2026-10-09T23:58:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T00:08:50.735749444Z","timing_metadata_checked_at":"2026-10-10T00:08:50.735749444Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T00:08:50.740665629Z","region_metadata_checked_at":"2026-10-10T00:08:50.740665629Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2076117d182d0c6ff07412458edf63de82e3fa8046b744070e6b2b81f44f701a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108673315018117120","occurred_at":"2026-10-09T21:37:50Z","failed_at":"e2e","signature_id":"cf36901b570ac1dee52b2e3df8bdc060e95f8945a01b9c43a720ca81ce35091f","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 651.764s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x104322d955a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[23:00:07.094]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452\"\n }�[0m\n �[37m[23:00:10.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:10.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 72/289 failed (KQL: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:10.102]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\"\n }�[0m\n �[37m[23:00:10.102]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:10.310]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:10.310]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\"\n }�[0m\n �[37m[23:00:10.320]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:20.724]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n \"commits\": 294\n }�[0m\n �[37m[23:00:20.724]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:20.786]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:20.857]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:21.018]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:21.198]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:21.347]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:00:21.884]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452\"\n }�[0m\n �[37m[23:00:21.884]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[23:00:22.064]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\"\n }�[0m\n �[37m[23:00:24.931]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:24.931]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 54/289 failed (KQL: let _startTime = datetime(2026-10-09T03:25:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T03:30:22.0635261Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:24.965]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\"\n }�[0m\n �[37m[23:00:24.965]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:25.188]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:25.188]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\"\n }�[0m\n �[37m[23:00:25.198]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:31.232]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n \"commits\": 294\n }�[0m\n �[37m[23:00:31.233]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:31.296]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:31.363]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:31.520]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:31.665]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:31.785]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:00:32.084]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\"\n }�[0m\n �[37m[23:00:32.084]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\"\n }�[0m\n \n query-skip-kubernetes-events: command failed: exit status 1\n output: �[37m[23:00:32.265]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\"\n }�[0m\n �[37m[23:00:35.196]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:35.196]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 73/289 failed (KQL: let _startTime = datetime(2026-10-09T05:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T05:05:32.2650856Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:35.196]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\"\n }�[0m\n �[37m[23:00:35.196]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:35.406]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:35.406]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\"\n }�[0m\n �[37m[23:00:35.416]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:41.551]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n \"commits\": 294\n }�[0m\n �[37m[23:00:41.551]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:41.611]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:41.684]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:41.712]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:41.713]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:41.713]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:41.839]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:41.983]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:42.104]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:00:42.413]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\"\n }�[0m\n �[37m[23:00:42.413]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[23:00:42.585]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\"\n }�[0m\n �[37m[23:00:46.166]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:46.166]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 74/289 failed (KQL: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:46.229]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\"\n }�[0m\n �[37m[23:00:46.229]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:46.437]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:46.437]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\"\n }�[0m\n �[37m[23:00:46.447]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:49.188]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:49.244]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:49.310]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:49.455]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:49.591]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:49.701]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:00:49.989]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\"\n }�[0m\n �[37m[23:00:49.989]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n }�[0m\n \n query-skip-hcp-and-events: command failed: exit status 1\n output: �[37m[23:00:50.121]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\"\n }�[0m\n �[37m[23:00:53.109]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:53.110]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 70/289 failed (KQL: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:53.110]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\"\n }�[0m\n �[37m[23:00:53.110]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:53.326]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:53.326]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\"\n }�[0m\n �[37m[23:00:53.335]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:59.329]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n \"commits\": 294\n }�[0m\n �[37m[23:00:59.329]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:59.389]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:59.462]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:59.614]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:59.757]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:59.878]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:01:00.172]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\"\n }�[0m\n �[37m[23:01:00.173]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\"\n }�[0m\n \n {\n s: \"must-gather CLI test failures:\\n query-basic: command failed: exit status 1\\noutput: \\x1b[37m[23:00:07.094]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.068]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.068]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 72/289 failed (KQL: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.102]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.102]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.320]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:20.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.786]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.857]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.018]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.198]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.347]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.884]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.884]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n query-skip-hcp-logs: command failed: exit status 1\\noutput: \\x1b[37m[23:00:22.064]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.931]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.931]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 54/289 failed (KQL: let _startTime = datetime(2026-10-09T03:25:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T03:30:22.0635261Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.965]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.965]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.198]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:31.232]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.233]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.296]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.363]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.520]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.665]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.785]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:32.084]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:32.084]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n query-skip-kubernetes-events: command failed: exit status 1\\noutput: \\x1b[37m[23:00:32.265]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 73/289 failed (KQL: let _startTime = datetime(2026-10-09T05:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T05:05:32.2650856Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.406]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.406]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.416]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:41.551]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.551]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.611]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.684]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.712]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.713]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.713]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.839]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.983]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.104]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.413]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.413]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n query-collect-systemd-logs: command failed: exit status 1\\noutput: \\x1b[37m[23:00:42.585]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.166]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.166]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 74/289 failed (KQL: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.229]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.229]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.447]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:49.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.244]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.455]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.591]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.701]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.989]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.989]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n query-skip-hcp-and-events: command failed: exit status 1\\noutput: \\x1b[37m[23:00:50.121]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.109]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 70/289 failed (KQL: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.335]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:59.329]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.329]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.389]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.462]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.614]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.757]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.878]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:01:00.172]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:01:00.173]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 651.764s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x104322d955a0\u003e: \n must-gather CLI test failures:\n query-basic: command failed: exit status 1\n output: �[37m[23:00:07.094]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452\"\n }�[0m\n �[37m[23:00:10.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:10.068]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 72/289 failed (KQL: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:10.102]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\"\n }�[0m\n �[37m[23:00:10.102]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:10.310]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:10.310]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\"\n }�[0m\n �[37m[23:00:10.320]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:20.724]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n \"commits\": 294\n }�[0m\n �[37m[23:00:20.724]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:20.786]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:20.857]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:20.886]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:21.018]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:21.198]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:21.347]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:00:21.884]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-basic-3825519452\"\n }�[0m\n �[37m[23:00:21.884]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:08.6161084Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\", ActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", SubActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\"}, IsPermanent:false})\"\n }�[0m\n \n query-skip-hcp-logs: command failed: exit status 1\n output: �[37m[23:00:22.064]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\"\n }�[0m\n �[37m[23:00:24.931]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:24.931]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 54/289 failed (KQL: let _startTime = datetime(2026-10-09T03:25:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T03:30:22.0635261Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:24.965]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\"\n }�[0m\n �[37m[23:00:24.965]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:25.188]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:25.188]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\"\n }�[0m\n �[37m[23:00:25.198]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:31.232]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n \"commits\": 294\n }�[0m\n �[37m[23:00:31.233]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:31.296]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:31.363]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:31.391]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:31.520]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:31.665]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:31.785]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:00:32.084]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\"\n }�[0m\n �[37m[23:00:32.084]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:23.5364121Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\", ActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", SubActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\"}, IsPermanent:false})\"\n }�[0m\n \n query-skip-kubernetes-events: command failed: exit status 1\n output: �[37m[23:00:32.265]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\"\n }�[0m\n �[37m[23:00:35.196]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:35.196]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 73/289 failed (KQL: let _startTime = datetime(2026-10-09T05:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T05:05:32.2650856Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:35.196]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\"\n }�[0m\n �[37m[23:00:35.196]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:35.406]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:35.406]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\"\n }�[0m\n �[37m[23:00:35.416]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:41.551]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n \"commits\": 294\n }�[0m\n �[37m[23:00:41.551]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:41.611]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:41.684]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:41.712]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:41.713]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:41.713]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:41.839]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:41.983]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:42.104]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:00:42.413]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\"\n }�[0m\n �[37m[23:00:42.413]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:33.7819049Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\", ActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", SubActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\"}, IsPermanent:false})\"\n }�[0m\n \n query-collect-systemd-logs: command failed: exit status 1\n output: �[37m[23:00:42.585]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\"\n }�[0m\n �[37m[23:00:46.166]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:46.166]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 74/289 failed (KQL: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:46.229]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\"\n }�[0m\n �[37m[23:00:46.229]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:46.437]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:46.437]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\"\n }�[0m\n �[37m[23:00:46.447]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:49.188]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:49.244]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:49.310]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:49.336]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:49.455]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:49.591]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:49.701]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:00:49.989]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\"\n }�[0m\n �[37m[23:00:49.989]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:44.8927968Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\", ActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", SubActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\"}, IsPermanent:false})\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\n\"\n }�[0m\n \n query-skip-hcp-and-events: command failed: exit status 1\n output: �[37m[23:00:50.121]�[0m �[36mINFO:�[0m �[97mgathering must-gather logs�[0m �[90m{\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\"\n }�[0m\n �[37m[23:00:53.109]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\",\n \"name\": \"detailedInfraOrchestrationLogs\"\n }�[0m\n �[37m[23:00:53.110]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n \"err\": \"query \\\"backendControllerConditions\\\" time window 70/289 failed (KQL: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\ndatabase('ServiceLogs').table('backendLogs')\\n| where timestamp between (_startTime .. _endTime)\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\n| where container_name == 'aro-hcp-backend'\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\n| extend controller_name = extract(\\\"/hcpOpenShiftControllers/([^\\\\\\\\/]+)\\\", 1, tostring(log.currentResourceID))\\n| mv-expand condition = log.content.properties.status.conditions\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context canceled\",\n \"name\": \"backendControllerConditions\"\n }�[0m\n �[37m[23:00:53.110]�[0m �[36mINFO:�[0m �[97mgenerating Cosmos content Git history�[0m �[90m{\n \"input\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\"\n }�[0m\n �[37m[23:00:53.110]�[0m �[36mINFO:�[0m �[97mParsing log file�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\"\n }�[0m\n �[37m[23:00:53.326]�[0m �[36mINFO:�[0m �[97mFound data dump entries�[0m �[90m{\n \"count\": 611,\n \"files\": 1\n }�[0m\n �[37m[23:00:53.326]�[0m �[36mINFO:�[0m �[97mInitializing git repository�[0m �[90m{\n \"path\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\"\n }�[0m\n �[37m[23:00:53.335]�[0m �[36mINFO:�[0m �[97mProcessing entries and creating commits�[0m �[90m{}�[0m\n �[37m[23:00:59.329]�[0m �[36mINFO:�[0m �[97mCompleted�[0m �[90m{\n \"commits\": 294\n }�[0m\n �[37m[23:00:59.329]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for clusters in the resource group�[0m �[90m{\n \"clusterIDs\": null,\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect\"\n }�[0m\n �[37m[23:00:59.389]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: cluster IDs to inspect�[0m �[90m{\n \"clusterIDs\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\"\n ]\n }�[0m\n �[37m[23:00:59.462]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered infra cluster names�[0m �[90m{\n \"clusterNames\": [\n \"ci01-j8117120-mgmt-1\",\n \"ci01-j8117120-svc\"\n ]\n }�[0m\n �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaces\": [\n \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n ]\n }�[0m\n �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97moc-adm-inspect: discovered namespaces for resource group clusters�[0m �[90m{\n \"managementCluster\": \"ci01-j8117120-mgmt-1\",\n \"namespaceCount\": 4\n }�[0m\n �[37m[23:00:59.489]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:59.614]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:59.757]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\"\n }�[0m\n �[37m[23:00:59.878]�[0m �[36mINFO:�[0m �[97mrunning oc-adm-inspect for namespace�[0m �[90m{\n \"cluster\": \"ci01-j8117120-mgmt-1\",\n \"namespace\": \"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\"\n }�[0m\n �[37m[23:01:00.172]�[0m �[91mERROR:�[0m �[97mmust-gather finished with errors; partial content may have been written�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\",\n \"output\": \"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\"\n }�[0m\n �[37m[23:01:00.173]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-09T23:00:51.6990414Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\", ActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", SubActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\"}, IsPermanent:false})\"\n }�[0m\n \n {\n s: \"must-gather CLI test failures:\\n query-basic: command failed: exit status 1\\noutput: \\x1b[37m[23:00:07.094]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.068]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.068]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 72/289 failed (KQL: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:55:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T05:00:07.0938714Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.102]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.102]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:10.320]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:20.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.724]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.786]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.857]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:20.886]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.018]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.198]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.347]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.884]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-basic-3825519452\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:21.884]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:07.0938715Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:07.0938720Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:08.6161084Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0\\\\\\\", ActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", SubActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"fa9fdc45-fd1f-45a6-b92b-0c4aefd52007\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;449095d6-2662-47ac-b98c-84b1c221c4c0 ARID=fa9fdc45-fd1f-45a6-b92b-0c4aefd52007 \\\\u003e GW.Http.CallContext/fa9fdc45-fd1f-45a6-b92b-0c4aefd52007)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n query-skip-hcp-logs: command failed: exit status 1\\noutput: \\x1b[37m[23:00:22.064]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.931]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.931]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 54/289 failed (KQL: let _startTime = datetime(2026-10-09T03:25:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T03:30:22.0635261Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.965]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:24.965]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:25.198]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:31.232]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.233]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.296]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.363]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.391]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.520]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.665]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:31.785]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:32.084]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-logs-2729146616\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:32.084]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:22.0635262Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:22.0635266Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:23.5364121Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf\\\\\\\", ActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", SubActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;f7de9eb3-7a31-483b-a7f9-1bd521db66cf ARID=98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd \\\\u003e GW.Http.CallContext/98bd0dee-a5dd-4e86-b8b2-3f1c9d52f1dd)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n query-skip-kubernetes-events: command failed: exit status 1\\noutput: \\x1b[37m[23:00:32.265]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 73/289 failed (KQL: let _startTime = datetime(2026-10-09T05:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T05:05:32.2650856Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.196]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.406]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.406]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:35.416]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:41.551]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.551]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.611]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.684]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.712]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.713]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.713]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.839]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:41.983]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.104]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.413]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-kubernetes-events-352999048\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:42.413]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:32.2650857Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:32.2650859Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:33.7819049Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0\\\\\\\", ActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", SubActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"1cb1bb92-57cf-464f-bf00-48034d2d6a93\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;01e664e9-6d59-4a47-8248-2db95e91d8b0 ARID=1cb1bb92-57cf-464f-bf00-48034d2d6a93 \\\\u003e GW.Http.CallContext/1cb1bb92-57cf-464f-bf00-48034d2d6a93)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\\n query-collect-systemd-logs: command failed: exit status 1\\noutput: \\x1b[37m[23:00:42.585]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.166]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.166]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 74/289 failed (KQL: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T05:05:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T05:10:42.5848875Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.229]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.229]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.437]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:46.447]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:49.188]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.244]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.310]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.336]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.455]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.591]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.701]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.989]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-collect-systemd-logs-3798035649\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:49.989]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:42.5848876Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:42.5848880Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:44.8927968Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb\\\\\\\", ActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", SubActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"5d767d80-6ecf-4205-a662-cd03a907247f\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;d3575990-ea88-461a-9367-5a48a83ce5bb ARID=5d767d80-6ecf-4205-a662-cd03a907247f \\\\u003e GW.Http.CallContext/5d767d80-6ecf-4205-a662-cd03a907247f)\\\\\\\"}, IsPermanent:false})\\\\nfailed to generate Cosmos content Git history: failed to process entries: failed to stage file: fork/exec /usr/bin/git: resource temporarily unavailable\\\\n\\\"\\n}\\x1b[0m\\n\\n query-skip-hcp-and-events: command failed: exit status 1\\noutput: \\x1b[37m[23:00:50.121]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgathering must-gather logs\\x1b[0m \\x1b[90m{\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.109]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"name\\\": \\\"detailedInfraOrchestrationLogs\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mQuery failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"query \\\\\\\"backendControllerConditions\\\\\\\" time window 70/289 failed (KQL: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message): failed to execute query: Op(OpQuery): Kind(KHTTPError): With query: let _startTime = datetime(2026-10-09T04:45:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T04:50:50.1214503Z);\\\\ndatabase('ServiceLogs').table('backendLogs')\\\\n| where timestamp between (_startTime .. _endTime)\\\\n| where cluster has_any ('ci01-j8117120-mgmt-1', 'ci01-j8117120-svc', 'ci01-j8117120-mgmt-1')\\\\n| where container_name == 'aro-hcp-backend'\\\\n| where log.msg has 'dumping resourceID' and log.msg has 'hcpOpenShiftControllers'\\\\n| extend controller_name = extract(\\\\\\\"/hcpOpenShiftControllers/([^\\\\\\\\\\\\\\\\/]+)\\\\\\\", 1, tostring(log.currentResourceID))\\\\n| mv-expand condition = log.content.properties.status.conditions\\\\n| project timestamp, log, cluster, condition.lastTransitionTime, controller_name, condition.type, condition.status, condition.reason, condition.message, Post \\\\\\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\\\\\": context canceled\\\",\\n \\\"name\\\": \\\"backendControllerConditions\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mgenerating Cosmos content Git history\\x1b[0m \\x1b[90m{\\n \\\"input\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.110]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mParsing log file\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/custom/custom-query_cosmosResourceSnapshots.jsonl\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mFound data dump entries\\x1b[0m \\x1b[90m{\\n \\\"count\\\": 611,\\n \\\"files\\\": 1\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.326]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mInitializing git repository\\x1b[0m \\x1b[90m{\\n \\\"path\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/cosmosContent\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:53.335]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mProcessing entries and creating commits\\x1b[0m \\x1b[90m{}\\x1b[0m\\n\\x1b[37m[23:00:59.329]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mCompleted\\x1b[0m \\x1b[90m{\\n \\\"commits\\\": 294\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.329]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for clusters in the resource group\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": null,\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.389]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: cluster IDs to inspect\\x1b[0m \\x1b[90m{\\n \\\"clusterIDs\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.462]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered infra cluster names\\x1b[0m \\x1b[90m{\\n \\\"clusterNames\\\": [\\n \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"ci01-j8117120-svc\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaces\\\": [\\n \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n ]\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97moc-adm-inspect: discovered namespaces for resource group clusters\\x1b[0m \\x1b[90m{\\n \\\"managementCluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespaceCount\\\": 4\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.489]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.614]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/klusterlet-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.757]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:00:59.878]\\x1b[0m \\x1b[36mINFO:\\x1b[0m \\x1b[97mrunning oc-adm-inspect for namespace\\x1b[0m \\x1b[90m{\\n \\\"cluster\\\": \\\"ci01-j8117120-mgmt-1\\\",\\n \\\"namespace\\\": \\\"ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126/oc-adm-inspect/namespaces/ocm-arohcpci01-2tbhqli0e1h6selklh84pg26u47hn0qa-g0r0h0s1i0y5q0i\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:01:00.172]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mmust-gather finished with errors; partial content may have been written\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\",\\n \\\"output\\\": \\\"/tmp/must-gather-cli-e2e-query-skip-hcp-and-events-94089126\\\"\\n}\\x1b[0m\\n\\x1b[37m[23:01:00.173]\\x1b[0m \\x1b[91mERROR:\\x1b[0m \\x1b[97mcommand failed\\x1b[0m \\x1b[90m{\\n \\\"err\\\": \\\"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\\\\\"detailedInfraOrchestrationLogs\\\\\\\" result (KQL: let _startTime = datetime(2026-10-08T23:00:50.1214504Z);\\\\nlet _endTime = datetime(2026-10-09T23:00:50.1214507Z);\\\\n// Extract cluster ID from CS logs using cid field\\\\nlet _cs_cluster_id = toscalar(\\\\n database('ServiceLogs').table('clustersServiceLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where resource_id has 'kusto-logs-5qwwl8d6wvg6'\\\\n | where isnotempty(log.cid)\\\\n | distinct tostring(log.cid)\\\\n | limit 1\\\\n);\\\\n// Find all Maestro resource IDs for this cluster\\\\nlet maestro_resource_ids = (\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | extend resourceId = extract(@'resourceid[=:\\\\\\\"]+([a-f0-9\\\\\\\\-]+)', 1, logStr)\\\\n | where isnotempty(resourceId)\\\\n | distinct resourceId\\\\n);\\\\n// Infrastructure \\\\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\\\nunion\\\\n (\\\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"maestro\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\\\n // Includes the CR reconciliation logs\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"hypershift\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n ),\\\\n (\\\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\\\n // Includes: work agent operations, manifest application, status feedback\\\\n database('ServiceLogs').table('containerLogs')\\\\n | where timestamp between (_startTime .. _endTime)\\\\n | where namespace_name has \\\\\\\"open-cluster-management-agent\\\\\\\"\\\\n | extend logStr = tostring(log)\\\\n | where logStr has _cs_cluster_id\\\\n | project timestamp, container_name, msg = logStr, log\\\\n )\\\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\\\\\"LimitsExceeded\\\\\\\", Message:\\\\\\\"Request is invalid and cannot be executed.\\\\\\\", Description:\\\\\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\\\\\", Type:\\\\\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\\\\\", Context:v2.ErrorContext{Timestamp:\\\\\\\"2026-10-09T23:00:51.6990414Z\\\\\\\", ServiceAlias:\\\\\\\"HCP-DEV-US-2\\\\\\\", MachineName:\\\\\\\"\\\\\\\", ProcessName:\\\\\\\"\\\\\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\\\\\"KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb\\\\\\\", ActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", SubActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityType:\\\\\\\"GW.Http.CallContext\\\\\\\", ParentActivityId:\\\\\\\"6c1e30d0-9a19-434e-a986-7816b0e42259\\\\\\\", ActivityStack:\\\\\\\"(Activity stack: CRID=KGC.execute;faa1700e-f80f-400f-b4c2-b40b4bbdddbb ARID=6c1e30d0-9a19-434e-a986-7816b0e42259 \\\\u003e GW.Http.CallContext/6c1e30d0-9a19-434e-a986-7816b0e42259)\\\\\\\"}, IsPermanent:false})\\\"\\n}\\x1b[0m\\n\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1"}},{"row_id":"21ac2d8d903eaf2c71f39f81ebfcb7b94a864d5afb83ce22a881cf3345223d83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108673315018117120","occurred_at":"2026-10-09T21:37:50Z","failed_at":"alert","signature_id":"7d1a3870cd1a33572d8e26c0f1e350eca9e57ea685800f4e196987e73f6c5d34","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T23:35:06Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j8117120-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.216:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8117120/providers/microsoft.monitor/accounts/services-j8117120\", namespace=\"klusterlet-2tbhpus9mi4ns169dgekasr6fitd43j9\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-bc7zx\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tbhpus9mi4ns169dgekasr6fitd43j9 on cluster ci01-j8117120-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"b86ba7ffc52ed48bbb6ebf08c0e6b6af3e58657dbf62378e510c3d5a7fb244de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108673315018117120","occurred_at":"2026-10-09T21:37:50Z","failed_at":"e2e","signature_id":"ea5426c1ae50dc34a6a959cb8f4ddaf157e2c2b692d50dd5d9b193626990f1eb","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"77f90 sp=0x3ae32ce77ed0 pc=0x586c752\nnet/http.(*Transport).startDialConnForLocked.func1()\n\t/usr/lib/golang/src/net/http/transport.go:1629 +0x35 fp=0x3ae32ce77fe0 sp=0x3ae32ce77f90 pc=0x586c595\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x3ae32ce77fe8 sp=0x3ae32ce77fe0 pc=0x5540381\ncreated by net/http.(*Transport).startDialConnForLocked in goroutine 119\n\t/usr/lib/golang/src/net/http/transport.go:1628 +0x112\n\ngoroutine 167 gp=0x3ae32ce014a0 m=nil [semacquire]:\nruntime.gopark(0x3f00400063cb02?, 0x3ae32da24402?, 0xe0?, 0xc0?, 0x3ae32d696bb8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x3ae32d6969c0 sp=0x3ae32d6969a0 pc=0x553818e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x16f798a4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x3ae32d696a28 sp=0x3ae32d6969c0 pc=0x5513872\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x3ae32d696ac8 sp=0x3ae32d696a28 pc=0x54d6232\nruntime.gcAssistAlloc(0x3ae32ce014a0)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x3ae32d696b58 sp=0x3ae32d696ac8 pc=0x54dadd6\nruntime.deductAssistCredit(0x1?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x3ae32d696b78 sp=0x3ae32d696b58 pc=0x54cb6a6\nruntime.mallocgc(0x590, 0xf965760, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x3ae32d696ba8 sp=0x3ae32d696b78 pc=0x5534e67\nruntime.newobject(0x3ae32da2445d?)\n\t/usr/lib/golang/src/runtime/malloc.go:2157 +0x25 fp=0x3ae32d696bd0 sp=0x3ae32d696ba8 pc=0x54cb7a5\ncrypto/x509.parseCertificate({0x3ae32da4c000, 0x56d, 0x582})\n\t/usr/lib/golang/src/crypto/x509/parser.go:887 +0x45 fp=0x3ae32d697068 sp=0x3ae32d696bd0 pc=0x5729285\ncrypto/x509.ParseCertificate({0x3ae32da4c000?, 0x56d, 0x2ea83?})\n\t/usr/lib/golang/src/crypto/x509/parser.go:1077 +0x1d fp=0x3ae32d697090 sp=0x3ae32d697068 pc=0x572a69d\ncrypto/x509.(*CertPool).AppendCertsFromPEM(0x3ae32d4196e0, {0x3ae32da80000?, 0x100a51a0?, 0x10059470?})\n\t/usr/lib/golang/src/crypto/x509/cert_pool.go:231 +0x125 fp=0x3ae32d6971b0 sp=0x3ae32d697090 pc=0x571d2c5\ncrypto/x509.loadSystemRoots()\n\t/usr/lib/golang/src/crypto/x509/root_unix.go:72 +0x391 fp=0x3ae32d697298 sp=0x3ae32d6971b0 pc=0x572ccf1\ncrypto/x509.initSystemRoots()\n\t/usr/lib/golang/src/crypto/x509/root.go:43 +0x65 fp=0x3ae32d6972d8 sp=0x3ae32d697298 pc=0x572c785\nsync.(*Once).doSlow(0x54ca9ea?, 0x1f1a89e3ef160b9?)\n\t/usr/lib/golang/src/sync/once.go:78 +0xac fp=0x3ae32d697330 sp=0x3ae32d6972d8 pc=0x555198c\nsync.(*Once).Do(...)\n\t/usr/lib/golang/src/sync/once.go:69\ncrypto/x509.systemRootsPool()\n\t/usr/lib/golang/src/crypto/x509/root.go:32 +0x45 fp=0x3ae32d697380 sp=0x3ae32d697330 pc=0x572c645\ncrypto/x509.(*Certificate).Verify(0x3ae32d4ae008, {{0x3ae32d1d9be0, 0x19}, 0x3ae32d419620, 0x0, {0xc2aa7a68520d364e, 0xb6db3cc, 0x16f76d00}, {0x0, 0x0, ...}, ...})\n\t/usr/lib/golang/src/crypto/x509/verify.go:578 +0x18b fp=0x3ae32d697530 sp=0x3ae32d697380 pc=0x572f5cb\ncrypto/tls.(*Conn).verifyServerCertificate(0x3ae32d432008, {0x3ae32d49a5a0, 0x3, 0x4})\n\t/usr/lib/golang/src/crypto/tls/handshake_client.go:1168 +0xb0e fp=0x3ae32d6978b8 sp=0x3ae32d697530 pc=0x5781dee\ncrypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0x3ae32d697c80)\n\t/usr/lib/golang/src/crypto/tls/handshake_client_tls13.go:640 +0x29a fp=0x3ae32d697ab0 sp=0x3ae32d6978b8 pc=0x578629a\ncrypto/tls.(*clientHandshakeStateTLS13).handshake(0x3ae32d697c80)\n\t/usr/lib/golang/src/crypto/tls/handshake_client_tls13.go:141 +0x7cc fp=0x3ae32d697b70 sp=0x3ae32d697ab0 pc=0x578344c\ncrypto/tls.(*Conn).clientHandshake(0x3ae32d432008, {0x1018a380, 0x3ae32ca24690})\n\t/usr/lib/golang/src/crypto/tls/handshake_client.go:339 +0x828 fp=0x3ae32d697d88 sp=0x3ae32d697b70 pc=0x577cb68\ncrypto/tls.(*Conn).clientHandshake-fm({0x1018a380?, 0x3ae32ca24690?})\n\t\u003cautogenerated\u003e:1 +0x2a fp=0x3ae32d697db0 sp=0x3ae32d697d88 pc=0x57bcaca\ncrypto/tls.(*Conn).handshakeContext(0x3ae32d432008, {0x1018a380, 0x3ae32c9f4460})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1562 +0x329 fp=0x3ae32d697f70 sp=0x3ae32d697db0 pc=0x5773829\ncrypto/tls.(*Conn).HandshakeContext(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:1516\nnet/http.(*persistConn).addTLS.func2()\n\t/usr/lib/golang/src/net/http/transport.go:1736 +0x6e fp=0x3ae32d697fe0 sp=0x3ae32d697f70 pc=0x586d3ce\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x3ae32d697fe8 sp=0x3ae32d697fe0 pc=0x5540381\ncreated by net/http.(*persistConn).addTLS in goroutine 120\n\t/usr/lib/golang/src/net/http/transport.go:1732 +0x309\n\nrax 0x0\nrbx 0x7fe37a988640\nrcx 0x7fe3c265d94c\nrdx 0x6\nrdi 0xa6f0\nrsi 0xa6f4\nrbp 0xa6f4\nrsp 0x7fe37a987790\nr8 0x7fe37a987860\nr9 0x7fe3c27834e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fe37a987b60\nr14 0x3ae32bf97860\nr15 0xffffffffffffffff\nrip 0x7fe3c265d94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n77f90 sp=0x3ae32ce77ed0 pc=0x586c752\nnet/http.(*Transport).startDialConnForLocked.func1()\n\t/usr/lib/golang/src/net/http/transport.go:1629 +0x35 fp=0x3ae32ce77fe0 sp=0x3ae32ce77f90 pc=0x586c595\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x3ae32ce77fe8 sp=0x3ae32ce77fe0 pc=0x5540381\ncreated by net/http.(*Transport).startDialConnForLocked in goroutine 119\n\t/usr/lib/golang/src/net/http/transport.go:1628 +0x112\n\ngoroutine 167 gp=0x3ae32ce014a0 m=nil [semacquire]:\nruntime.gopark(0x3f00400063cb02?, 0x3ae32da24402?, 0xe0?, 0xc0?, 0x3ae32d696bb8?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x3ae32d6969c0 sp=0x3ae32d6969a0 pc=0x553818e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.semacquire1(0x16f798a4, 0x0, 0x0, 0x0, 0xd)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x232 fp=0x3ae32d696a28 sp=0x3ae32d6969c0 pc=0x5513872\nruntime.semacquire(...)\n\t/usr/lib/golang/src/runtime/sema.go:143\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1018 +0x32 fp=0x3ae32d696ac8 sp=0x3ae32d696a28 pc=0x54d6232\nruntime.gcAssistAlloc(0x3ae32ce014a0)\n\t/usr/lib/golang/src/runtime/mgcmark.go:647 +0x2f6 fp=0x3ae32d696b58 sp=0x3ae32d696ac8 pc=0x54dadd6\nruntime.deductAssistCredit(0x1?)\n\t/usr/lib/golang/src/runtime/malloc.go:1861 +0x46 fp=0x3ae32d696b78 sp=0x3ae32d696b58 pc=0x54cb6a6\nruntime.mallocgc(0x590, 0xf965760, 0x1)\n\t/usr/lib/golang/src/runtime/malloc.go:1113 +0x67 fp=0x3ae32d696ba8 sp=0x3ae32d696b78 pc=0x5534e67\nruntime.newobject(0x3ae32da2445d?)\n\t/usr/lib/golang/src/runtime/malloc.go:2157 +0x25 fp=0x3ae32d696bd0 sp=0x3ae32d696ba8 pc=0x54cb7a5\ncrypto/x509.parseCertificate({0x3ae32da4c000, 0x56d, 0x582})\n\t/usr/lib/golang/src/crypto/x509/parser.go:887 +0x45 fp=0x3ae32d697068 sp=0x3ae32d696bd0 pc=0x5729285\ncrypto/x509.ParseCertificate({0x3ae32da4c000?, 0x56d, 0x2ea83?})\n\t/usr/lib/golang/src/crypto/x509/parser.go:1077 +0x1d fp=0x3ae32d697090 sp=0x3ae32d697068 pc=0x572a69d\ncrypto/x509.(*CertPool).AppendCertsFromPEM(0x3ae32d4196e0, {0x3ae32da80000?, 0x100a51a0?, 0x10059470?})\n\t/usr/lib/golang/src/crypto/x509/cert_pool.go:231 +0x125 fp=0x3ae32d6971b0 sp=0x3ae32d697090 pc=0x571d2c5\ncrypto/x509.loadSystemRoots()\n\t/usr/lib/golang/src/crypto/x509/root_unix.go:72 +0x391 fp=0x3ae32d697298 sp=0x3ae32d6971b0 pc=0x572ccf1\ncrypto/x509.initSystemRoots()\n\t/usr/lib/golang/src/crypto/x509/root.go:43 +0x65 fp=0x3ae32d6972d8 sp=0x3ae32d697298 pc=0x572c785\nsync.(*Once).doSlow(0x54ca9ea?, 0x1f1a89e3ef160b9?)\n\t/usr/lib/golang/src/sync/once.go:78 +0xac fp=0x3ae32d697330 sp=0x3ae32d6972d8 pc=0x555198c\nsync.(*Once).Do(...)\n\t/usr/lib/golang/src/sync/once.go:69\ncrypto/x509.systemRootsPool()\n\t/usr/lib/golang/src/crypto/x509/root.go:32 +0x45 fp=0x3ae32d697380 sp=0x3ae32d697330 pc=0x572c645\ncrypto/x509.(*Certificate).Verify(0x3ae32d4ae008, {{0x3ae32d1d9be0, 0x19}, 0x3ae32d419620, 0x0, {0xc2aa7a68520d364e, 0xb6db3cc, 0x16f76d00}, {0x0, 0x0, ...}, ...})\n\t/usr/lib/golang/src/crypto/x509/verify.go:578 +0x18b fp=0x3ae32d697530 sp=0x3ae32d697380 pc=0x572f5cb\ncrypto/tls.(*Conn).verifyServerCertificate(0x3ae32d432008, {0x3ae32d49a5a0, 0x3, 0x4})\n\t/usr/lib/golang/src/crypto/tls/handshake_client.go:1168 +0xb0e fp=0x3ae32d6978b8 sp=0x3ae32d697530 pc=0x5781dee\ncrypto/tls.(*clientHandshakeStateTLS13).readServerCertificate(0x3ae32d697c80)\n\t/usr/lib/golang/src/crypto/tls/handshake_client_tls13.go:640 +0x29a fp=0x3ae32d697ab0 sp=0x3ae32d6978b8 pc=0x578629a\ncrypto/tls.(*clientHandshakeStateTLS13).handshake(0x3ae32d697c80)\n\t/usr/lib/golang/src/crypto/tls/handshake_client_tls13.go:141 +0x7cc fp=0x3ae32d697b70 sp=0x3ae32d697ab0 pc=0x578344c\ncrypto/tls.(*Conn).clientHandshake(0x3ae32d432008, {0x1018a380, 0x3ae32ca24690})\n\t/usr/lib/golang/src/crypto/tls/handshake_client.go:339 +0x828 fp=0x3ae32d697d88 sp=0x3ae32d697b70 pc=0x577cb68\ncrypto/tls.(*Conn).clientHandshake-fm({0x1018a380?, 0x3ae32ca24690?})\n\t\u003cautogenerated\u003e:1 +0x2a fp=0x3ae32d697db0 sp=0x3ae32d697d88 pc=0x57bcaca\ncrypto/tls.(*Conn).handshakeContext(0x3ae32d432008, {0x1018a380, 0x3ae32c9f4460})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1562 +0x329 fp=0x3ae32d697f70 sp=0x3ae32d697db0 pc=0x5773829\ncrypto/tls.(*Conn).HandshakeContext(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:1516\nnet/http.(*persistConn).addTLS.func2()\n\t/usr/lib/golang/src/net/http/transport.go:1736 +0x6e fp=0x3ae32d697fe0 sp=0x3ae32d697f70 pc=0x586d3ce\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x3ae32d697fe8 sp=0x3ae32d697fe0 pc=0x5540381\ncreated by net/http.(*persistConn).addTLS in goroutine 120\n\t/usr/lib/golang/src/net/http/transport.go:1732 +0x309\n\nrax 0x0\nrbx 0x7fe37a988640\nrcx 0x7fe3c265d94c\nrdx 0x6\nrdi 0xa6f0\nrsi 0xa6f4\nrbp 0xa6f4\nrsp 0x7fe37a987790\nr8 0x7fe37a987860\nr9 0x7fe3c27834e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fe37a987b60\nr14 0x3ae32bf97860\nr15 0xffffffffffffffff\nrip 0x7fe3c265d94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108664424637140992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7427,"pr_state":"open","pr_sha":"269733960011d9292bfded60a6aca75da02c364a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T21:02:30Z","started_at":"2026-10-09T21:02:30Z","completed_at":"2026-10-09T23:29:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T23:48:42.054110744Z","timing_metadata_checked_at":"2026-10-09T23:48:42.054110744Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T23:48:42.057797433Z","region_metadata_checked_at":"2026-10-09T23:48:42.057797433Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108661352737280000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7248,"pr_state":"open","pr_sha":"29ee7d9651963ed9c17688c509801cba7771db4c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T20:50:18Z","started_at":"2026-10-09T20:50:18Z","completed_at":"2026-10-09T23:23:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T23:37:53.311899109Z","timing_metadata_checked_at":"2026-10-09T23:37:53.311899109Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T23:37:53.316855295Z","region_metadata_checked_at":"2026-10-09T23:37:53.316855295Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"164dc2f50858e0e7dc8cbef175a97709ffc2382056368b3ddb76cb9e191d306e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108661352737280000","occurred_at":"2026-10-09T20:50:18Z","failed_at":"e2e","signature_id":"d6d78b4f553ecf032ff0039a772072e8dd415e92575979480f3e3627be38fdcf","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-rf2ccpxh56qx/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-rf2ccpxh56qx/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-8jfp6hmxz2vs/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108648932484583424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7227,"pr_state":"closed","pr_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","final_merged_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T20:00:53Z","started_at":"2026-10-09T20:00:53Z","completed_at":"2026-10-09T22:29:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T22:45:54.492080154Z","timing_metadata_checked_at":"2026-10-09T22:45:54.492080154Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T22:45:54.496156442Z","region_metadata_checked_at":"2026-10-09T22:45:54.496156442Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9c7466558fea7fa4024b0a250b0a0e25df80797af06c09d89059de4af4aaee77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108648932484583424","occurred_at":"2026-10-09T20:00:53Z","failed_at":"e2e","signature_id":"1bd05fb2c90549773c1175fd56546ae62c47d903fe566a61ea1dc83c30a119d8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-j7bk7p\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x76fa41aa1e0\u003e: \n DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x76fa41aa1a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-j7bk7p\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x76fa41aa1e0\u003e: \n DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x76fa41aa1a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108648873588166656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"closed","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T20:00:42Z","started_at":"2026-10-09T20:00:42Z","completed_at":"2026-10-09T22:31:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T22:45:54.516330585Z","timing_metadata_checked_at":"2026-10-09T22:45:54.516330585Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T22:45:54.519095077Z","region_metadata_checked_at":"2026-10-09T22:45:54.519095077Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1ef1988d69b0622f7afd1f3fa40003883c4816b92e6b32f9bf7765a2860017d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108648873588166656","occurred_at":"2026-10-09T20:00:42Z","failed_at":"alert","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-09T21:28:57Z\n Ended: 2026-10-09T21:36:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8166656-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"447cb27d04314f01839447bab7a26dd9f9bc195adf6fbe3d914cb45760290f63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108648873588166656","occurred_at":"2026-10-09T20:00:42Z","failed_at":"e2e","signature_id":"99c7ef686eed46a4a71bdbed72621290bd7f8e8a9969f168d079f9557b9819e9","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x247e648e4440\u003e: \n DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x247e648e4420\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x247e648e4440\u003e: \n DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x247e648e4420\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9xcxj.apps.aro.v20261001.jhke.j8166656.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108642958508036096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"closed","pr_sha":"f2342694ca433fce7c8afb88f010b598afa14625","final_merged_sha":"f2342694ca433fce7c8afb88f010b598afa14625","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T19:37:12Z","started_at":"2026-10-09T19:37:12Z","completed_at":"2026-10-09T21:52:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T22:04:00.948250249Z","timing_metadata_checked_at":"2026-10-09T22:04:00.948250249Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T22:04:00.952979536Z","region_metadata_checked_at":"2026-10-09T22:04:00.952979536Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108618645423788032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7427,"pr_state":"open","pr_sha":"269733960011d9292bfded60a6aca75da02c364a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T18:00:35Z","started_at":"2026-10-09T18:00:35Z","completed_at":"2026-10-09T20:26:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T20:39:48.892556946Z","timing_metadata_checked_at":"2026-10-09T20:39:48.892556946Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T20:39:48.897228832Z","region_metadata_checked_at":"2026-10-09T20:39:48.897228832Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cef18de65e3505883d7957def1b1997d8e0a09ab4fd069f9bfaeda06c4391a94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108618645423788032","occurred_at":"2026-10-09T18:00:35Z","failed_at":"e2e","signature_id":"ecb11b0d98366bb57461afb65e5a662d355410505007e7cb21a482baec673750","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x3113509f94c0\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-g776ncnttgrt\" to finish creating: GET https://rp.j3788032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77fe8352-5c4d-493e-9b0c-7d5a039edff5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77fe8352-5c4d-493e-9b0c-7d5a039edff5\",\n \"name\": \"77fe8352-5c4d-493e-9b0c-7d5a039edff5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T19:34:01.982763751Z\",\n \"endTime\": \"2026-10-09T20:03:10.160667249Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-5j5ks-29l5h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-29l5h to exist\\nMachine ephemeral-disk-ephemeral-np-5j5ks-4jtss: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-4jtss to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-5j5ks-29l5h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-29l5h to exist\\nMachine ephemeral-disk-ephemeral-np-5j5ks-4jtss: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-4jtss to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x3113509f94c0\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-g776ncnttgrt\" to finish creating: GET https://rp.j3788032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77fe8352-5c4d-493e-9b0c-7d5a039edff5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77fe8352-5c4d-493e-9b0c-7d5a039edff5\",\n \"name\": \"77fe8352-5c4d-493e-9b0c-7d5a039edff5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T19:34:01.982763751Z\",\n \"endTime\": \"2026-10-09T20:03:10.160667249Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-5j5ks-29l5h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-29l5h to exist\\nMachine ephemeral-disk-ephemeral-np-5j5ks-4jtss: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-4jtss to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-5j5ks-29l5h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-29l5h to exist\\nMachine ephemeral-disk-ephemeral-np-5j5ks-4jtss: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-g776ncnttgrt--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-5j5ks-4jtss to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6904/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108611968167317504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6904,"pr_state":"open","pr_sha":"d202f21693d4a1f475ddd77af519728ba84f938f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T17:34:03Z","started_at":"2026-10-09T17:34:03Z","completed_at":"2026-10-09T20:06:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T20:18:11.340141968Z","timing_metadata_checked_at":"2026-10-09T20:18:11.340141968Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T20:18:11.343882456Z","region_metadata_checked_at":"2026-10-09T20:18:11.343882456Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"222eeeefbbd1848239a83a3eb6fc367d000559f0aff3a028dc5e7bb36481daca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6904/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108611968167317504","occurred_at":"2026-10-09T17:34:03Z","failed_at":"alert","signature_id":"6fd1487b882a46fa816895ea3781fe05946c86b7aff2a9b326e2ac82c2028920","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T19:21:13Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7317504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.112:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7317504/providers/microsoft.monitor/accounts/hcps-j7317504\", namespace=\"ocm-arohcpci01-2tbe6sullvo3b9gl7hggkskdvnm94ofr-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8xgvb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbe6sullvo3b9gl7hggkskdvnm94ofr-basic-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T19:26:12Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7317504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.112:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7317504/providers/microsoft.monitor/accounts/hcps-j7317504\", namespace=\"ocm-arohcpci01-2tbe6sullvo3b9gl7hggkskdvnm94ofr-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8xgvb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbe6sullvo3b9gl7hggkskdvnm94ofr-basic-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T19:27:12Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7317504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"csi-snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.112:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7317504/providers/microsoft.monitor/accounts/hcps-j7317504\", namespace=\"ocm-arohcpci01-2tbe6sullvo3b9gl7hggkskdvnm94ofr-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8xgvb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbe6sullvo3b9gl7hggkskdvnm94ofr-basic-cluster/csi-snapshot-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108611234440941568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7444,"pr_state":"open","pr_sha":"4efbf6bf8baa7e1355ea06418637c2c256e8ccab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T17:31:09Z","started_at":"2026-10-09T17:31:09Z","completed_at":"2026-10-09T19:55:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T20:07:50.86804095Z","timing_metadata_checked_at":"2026-10-09T20:07:50.86804095Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T20:07:50.871918538Z","region_metadata_checked_at":"2026-10-09T20:07:50.871918538Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4e2669597fe24710456a4bad9bed64f02b50c70323bd4ecd8ddafbc9807ec0c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108611234440941568","occurred_at":"2026-10-09T17:31:09Z","failed_at":"e2e","signature_id":"7b2af89a3191d6534ece213e75e531df0a43a9214041bfedcb654f1eda579522","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-8jfp6hmxz2vs/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-8jfp6hmxz2vs/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-8jfp6hmxz2vs/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"open","pr_sha":"8a3c49af442e20571b90023b5ceed7d898443db7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T17:15:54Z","started_at":"2026-10-09T17:15:54Z","completed_at":"2026-10-09T19:33:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T19:46:54.493947116Z","timing_metadata_checked_at":"2026-10-09T19:46:54.493947116Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T19:46:54.497477706Z","region_metadata_checked_at":"2026-10-09T19:46:54.497477706Z"},"failed_at":["alert","e2e"],"failed_test_count":73,"occurrences":[{"row_id":"020850606ce51cc7705be086bf46db7d6f5eae996cad693cb404ad53fbecb468","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"99108240d9af947608575ec6e131c4e8ea683becdcb1c6901189bec0b24500ba","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b701c6046e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-jlldrxpj8hv8\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f084bf11-c7ee-4325-bb49-cf7c84015e14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f084bf11-c7ee-4325-bb49-cf7c84015e14\",\n \"name\": \"f084bf11-c7ee-4325-bb49-cf7c84015e14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:43.736446438Z\",\n \"endTime\": \"2026-10-09T18:31:48.075237736Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b701c6046e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-jlldrxpj8hv8\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f084bf11-c7ee-4325-bb49-cf7c84015e14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f084bf11-c7ee-4325-bb49-cf7c84015e14\",\n \"name\": \"f084bf11-c7ee-4325-bb49-cf7c84015e14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:43.736446438Z\",\n \"endTime\": \"2026-10-09T18:31:48.075237736Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0862edef72c0e53e41588b4955a059664210e4e4b035686393d84b0728342792","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"7a09fd0d26a1837bd2f82a8277d0513b55de1cfb905ad76289110b9aa7c6097b","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e4acf3c8880\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-qbq2q99hlvjq\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/84d3f991-bfdd-41a1-92f7-c169b2020c7a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/84d3f991-bfdd-41a1-92f7-c169b2020c7a\",\n \"name\": \"84d3f991-bfdd-41a1-92f7-c169b2020c7a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.700624897Z\",\n \"endTime\": \"2026-10-09T18:31:17.853992528Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e4acf3c8880\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-qbq2q99hlvjq\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/84d3f991-bfdd-41a1-92f7-c169b2020c7a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/84d3f991-bfdd-41a1-92f7-c169b2020c7a\",\n \"name\": \"84d3f991-bfdd-41a1-92f7-c169b2020c7a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.700624897Z\",\n \"endTime\": \"2026-10-09T18:31:17.853992528Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"09988995eca496ea3d9eb5f566107007ac7324e25a57773ac5d382401cac02ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"725401be84daab8b061390e9b75a1844bbed2e06bc3458add8fd0998fa58edf6","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f9b37b2d6a0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-cd2gbr7b2jq2\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6aa4636f-6812-4007-8149-5c1264bfc4d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6aa4636f-6812-4007-8149-5c1264bfc4d4\",\n \"name\": \"6aa4636f-6812-4007-8149-5c1264bfc4d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:14.537204498Z\",\n \"endTime\": \"2026-10-09T18:32:17.726086322Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f9b37b2d6a0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-cd2gbr7b2jq2\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6aa4636f-6812-4007-8149-5c1264bfc4d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6aa4636f-6812-4007-8149-5c1264bfc4d4\",\n \"name\": \"6aa4636f-6812-4007-8149-5c1264bfc4d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:14.537204498Z\",\n \"endTime\": \"2026-10-09T18:32:17.726086322Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0a4567e6c4fdfa38490399bbcb6a27d4df04c2fd4abec3db82f41536abd56900","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"0d88e33eff3b08810a358f00c8d58a2d311131200ec6c8ef38f667c486c55827","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3aa3a58fd700\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r9mpb5rglfkx\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/501ef370-ff82-48d0-af1d-38c7e8e27d51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/501ef370-ff82-48d0-af1d-38c7e8e27d51\",\n \"name\": \"501ef370-ff82-48d0-af1d-38c7e8e27d51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:09.983214688Z\",\n \"endTime\": \"2026-10-09T18:31:17.777246897Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x3aa3a58fd700\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-r9mpb5rglfkx\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/501ef370-ff82-48d0-af1d-38c7e8e27d51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/501ef370-ff82-48d0-af1d-38c7e8e27d51\",\n \"name\": \"501ef370-ff82-48d0-af1d-38c7e8e27d51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:09.983214688Z\",\n \"endTime\": \"2026-10-09T18:31:17.777246897Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0c5f56ec27c6be4daf92ac2801471c16153bfb71635617416e846eb7d3c15bc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"cd9e8e56479966f53ef3434d48eb0be4e3f969ddd83de89e192a9d68b26a3eae","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x1062442c6c90\u003e: \n GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/328cd7fa-dd20-4157-ae05-7528e1885422\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/328cd7fa-dd20-4157-ae05-7528e1885422\",\n \"name\": \"328cd7fa-dd20-4157-ae05-7528e1885422\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:24.978697652Z\",\n \"endTime\": \"2026-10-09T18:32:27.609289345Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x1062442c6c90\u003e: \n GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/328cd7fa-dd20-4157-ae05-7528e1885422\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/328cd7fa-dd20-4157-ae05-7528e1885422\",\n \"name\": \"328cd7fa-dd20-4157-ae05-7528e1885422\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:24.978697652Z\",\n \"endTime\": \"2026-10-09T18:32:27.609289345Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0c6809962db9bb88c14485e2e022dbc98118bc8c84eade8ccdd7851633e30970","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"1312f58b330d7732df9cff7e092e95b7ab0b3c27e718bf4dc5788182d045fd41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"86e6e152f41be537ce62e007989121fe49a0689d42902ba0575419da7d1cae09","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f0605478480\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-mng25fmg9dxg\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/979e4ed3-7c85-4832-a3e7-4f0677e8904e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/979e4ed3-7c85-4832-a3e7-4f0677e8904e\",\n \"name\": \"979e4ed3-7c85-4832-a3e7-4f0677e8904e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.582225717Z\",\n \"endTime\": \"2026-10-09T18:31:17.588996241Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f0605478480\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-mng25fmg9dxg\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/979e4ed3-7c85-4832-a3e7-4f0677e8904e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/979e4ed3-7c85-4832-a3e7-4f0677e8904e\",\n \"name\": \"979e4ed3-7c85-4832-a3e7-4f0677e8904e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.582225717Z\",\n \"endTime\": \"2026-10-09T18:31:17.588996241Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1356911f7c734972a1f5da351e6b64bb0f11685cc2176715f0f9c2d0b79a32b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"d94749aff98696c868c3dd015c137aa69ded1b9c88ae9994b2fbdea0a163c668","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e42295f70c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hlqs8nkdm5t5\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c2dd073-54b7-4e32-bb4f-1181788dad41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c2dd073-54b7-4e32-bb4f-1181788dad41\",\n \"name\": \"9c2dd073-54b7-4e32-bb4f-1181788dad41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:23.470629545Z\",\n \"endTime\": \"2026-10-09T18:31:27.909656234Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e42295f70c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hlqs8nkdm5t5\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c2dd073-54b7-4e32-bb4f-1181788dad41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c2dd073-54b7-4e32-bb4f-1181788dad41\",\n \"name\": \"9c2dd073-54b7-4e32-bb4f-1181788dad41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:23.470629545Z\",\n \"endTime\": \"2026-10-09T18:31:27.909656234Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"156db5809ae643f6ad673820ff52291dd0db056bfb6330691fd3e7357f6c298f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"bf0240e034eb5e7bef668792af253798232bc78025b0cfe8329b245ab54aff3e","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-5-1-mz7lls\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f661db12580\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-5-1-mz7lls: failed waiting for cluster=\"cp-ystream-deny-pods-5-1-mz7lls\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-1-msvnld9ccv56\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2df78d12-2cc9-405d-b31e-84090ef6cee2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2df78d12-2cc9-405d-b31e-84090ef6cee2\",\n \"name\": \"2df78d12-2cc9-405d-b31e-84090ef6cee2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:26.95261Z\",\n \"endTime\": \"2026-10-09T18:31:27.800516821Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-5-1-mz7lls\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f661db12580\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-5-1-mz7lls: failed waiting for cluster=\"cp-ystream-deny-pods-5-1-mz7lls\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-1-msvnld9ccv56\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2df78d12-2cc9-405d-b31e-84090ef6cee2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2df78d12-2cc9-405d-b31e-84090ef6cee2\",\n \"name\": \"2df78d12-2cc9-405d-b31e-84090ef6cee2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:26.95261Z\",\n \"endTime\": \"2026-10-09T18:31:27.800516821Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"166d44bac1935a31e66dbc0fba0914576d5d4291f10668db05003f0a049f68c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"697453e7f696f54f41edcb34030e48efa337b5e8853e8f54ed448226cf012cf3","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c6b96d827c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-znrkb7wxw8xs\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b57af5b-3bbd-4b3b-bb11-9deeae9a1edb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b57af5b-3bbd-4b3b-bb11-9deeae9a1edb\",\n \"name\": \"0b57af5b-3bbd-4b3b-bb11-9deeae9a1edb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:04.746790867Z\",\n \"endTime\": \"2026-10-09T19:10:05.371290727Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c6b96d827c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-znrkb7wxw8xs\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b57af5b-3bbd-4b3b-bb11-9deeae9a1edb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b57af5b-3bbd-4b3b-bb11-9deeae9a1edb\",\n \"name\": \"0b57af5b-3bbd-4b3b-bb11-9deeae9a1edb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:04.746790867Z\",\n \"endTime\": \"2026-10-09T19:10:05.371290727Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"167de1d699ea5a4a37e466e0f21336103758877721e014ddf5b2316baa17df54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"ac9d571735b7d2d819291b1cce9460a34edeb058e38bb2703a76fa82e279cdc1","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x6f04a2a4280\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2g2554z7wr65\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18e9c5d8-01c0-415e-b048-7b0270ff6b9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18e9c5d8-01c0-415e-b048-7b0270ff6b9e\",\n \"name\": \"18e9c5d8-01c0-415e-b048-7b0270ff6b9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:31.117011395Z\",\n \"endTime\": \"2026-10-09T19:09:35.538194291Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x6f04a2a4280\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2g2554z7wr65\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18e9c5d8-01c0-415e-b048-7b0270ff6b9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18e9c5d8-01c0-415e-b048-7b0270ff6b9e\",\n \"name\": \"18e9c5d8-01c0-415e-b048-7b0270ff6b9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:31.117011395Z\",\n \"endTime\": \"2026-10-09T19:09:35.538194291Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1be76c543533088e4341d1449ef2cd76eef3826d8982c8fe793e686c133f3e50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"80d08043ecfaf539ff53744c99fc856a641ca23589f93616ad5fd6281e5d2505","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x32c4d8d6b260\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-glcr4w5bpjtd\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ccad3244-2be6-4d98-b6eb-b17a3b23d168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ccad3244-2be6-4d98-b6eb-b17a3b23d168\",\n \"name\": \"ccad3244-2be6-4d98-b6eb-b17a3b23d168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.70429797Z\",\n \"endTime\": \"2026-10-09T18:31:17.793696001Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x32c4d8d6b260\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-glcr4w5bpjtd\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ccad3244-2be6-4d98-b6eb-b17a3b23d168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ccad3244-2be6-4d98-b6eb-b17a3b23d168\",\n \"name\": \"ccad3244-2be6-4d98-b6eb-b17a3b23d168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.70429797Z\",\n \"endTime\": \"2026-10-09T18:31:17.793696001Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1e3c552f15d567bdc0c476a4ac71b7e4e4d391415e183e5a3ebba14ef832b0e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"2ae8b16399c641a572db2f6a75461482166a2cd3b4fbba6fc168f57bfa5cb619","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x86a1799a760\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-fh4m5t8744rf\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b430aa9-d99a-445d-994d-9d7059d61cf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b430aa9-d99a-445d-994d-9d7059d61cf0\",\n \"name\": \"3b430aa9-d99a-445d-994d-9d7059d61cf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:05:45.777383606Z\",\n \"endTime\": \"2026-10-09T18:34:47.613925333Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x86a1799a760\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-fh4m5t8744rf\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b430aa9-d99a-445d-994d-9d7059d61cf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b430aa9-d99a-445d-994d-9d7059d61cf0\",\n \"name\": \"3b430aa9-d99a-445d-994d-9d7059d61cf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:05:45.777383606Z\",\n \"endTime\": \"2026-10-09T18:34:47.613925333Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1e5c9028629bc8ec21338b8eeb10591ce50d64e0516a9326011fc732969ced94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"af974a64f9672e06da3b97d10120baa2a2f2a5d5c7ea0999bd68dac78aaa5fac","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-hcccw7\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x8c4cacb2340\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-hcccw7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-hcccw7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-6xx9lxdctr9q\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18450837-6574-44f7-89e1-1540da6d0c6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18450837-6574-44f7-89e1-1540da6d0c6d\",\n \"name\": \"18450837-6574-44f7-89e1-1540da6d0c6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:28.52780907Z\",\n \"endTime\": \"2026-10-09T18:31:37.736612105Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-hcccw7\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x8c4cacb2340\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-hcccw7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-hcccw7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-6xx9lxdctr9q\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18450837-6574-44f7-89e1-1540da6d0c6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18450837-6574-44f7-89e1-1540da6d0c6d\",\n \"name\": \"18450837-6574-44f7-89e1-1540da6d0c6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:28.52780907Z\",\n \"endTime\": \"2026-10-09T18:31:37.736612105Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1fce936c52e08fc719083e499d3207fffdbc81859ff3369b076f46b116ce253d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"f266521e812d721d55035420d232767ea4a41f7eb1df70515690979a7dfbe6da","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-4-21-772t2t\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x17450f274740\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-4-21-772t2t: failed waiting for cluster=\"cp-ystream-deny-pods-4-21-772t2t\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-21-94fnnfh56glc\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2df55a27-9995-474e-8bc7-d399c8de9607\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2df55a27-9995-474e-8bc7-d399c8de9607\",\n \"name\": \"2df55a27-9995-474e-8bc7-d399c8de9607\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:28.540876037Z\",\n \"endTime\": \"2026-10-09T18:31:38.071799276Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-4-21-772t2t\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x17450f274740\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-4-21-772t2t: failed waiting for cluster=\"cp-ystream-deny-pods-4-21-772t2t\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-21-94fnnfh56glc\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2df55a27-9995-474e-8bc7-d399c8de9607\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2df55a27-9995-474e-8bc7-d399c8de9607\",\n \"name\": \"2df55a27-9995-474e-8bc7-d399c8de9607\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:28.540876037Z\",\n \"endTime\": \"2026-10-09T18:31:38.071799276Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"21b48ec7a6efae8442b0c8232a890d2abccc4f35dc47d4e689846643ac231e0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"823aa7b3881e8fb041db03af06d9fa85a03a6954f019b4555aaf30c8412e5a43","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xa4a6ab96900\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-gqg5dldzlbg2\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/23fed6b4-c8c3-47c9-b15d-8fac0bc11c61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/23fed6b4-c8c3-47c9-b15d-8fac0bc11c61\",\n \"name\": \"23fed6b4-c8c3-47c9-b15d-8fac0bc11c61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:23.028824085Z\",\n \"endTime\": \"2026-10-09T18:31:27.712062038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xa4a6ab96900\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-gqg5dldzlbg2\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/23fed6b4-c8c3-47c9-b15d-8fac0bc11c61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/23fed6b4-c8c3-47c9-b15d-8fac0bc11c61\",\n \"name\": \"23fed6b4-c8c3-47c9-b15d-8fac0bc11c61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:23.028824085Z\",\n \"endTime\": \"2026-10-09T18:31:27.712062038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2f41dfea860e893dc05b9fb42439d8adc1045087f3be2e5f30bc88f01574f003","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"ec4b90a129599d5500cd5af07d3433c687996c1b739d2913a2675d196f5b546e","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x63eb2f385c0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bht6w8mbz9cj\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f21fff17-63d9-4b47-bce9-ae469eda2373\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f21fff17-63d9-4b47-bce9-ae469eda2373\",\n \"name\": \"f21fff17-63d9-4b47-bce9-ae469eda2373\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:44.373195167Z\",\n \"endTime\": \"2026-10-09T19:09:45.384935766Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x63eb2f385c0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bht6w8mbz9cj\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f21fff17-63d9-4b47-bce9-ae469eda2373\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f21fff17-63d9-4b47-bce9-ae469eda2373\",\n \"name\": \"f21fff17-63d9-4b47-bce9-ae469eda2373\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:44.373195167Z\",\n \"endTime\": \"2026-10-09T19:09:45.384935766Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2f57841cb548cc38b2206e23a6efd9d61a26deb6782dfbaca28bfaa064facde6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"13de764af8574947f7d4d0670b47cfd74c8f2223eef88b15d28b542d71598982","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x199bbb710720\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-mrtmndgzlmdf\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b44366c-80d7-439e-998d-9fcc151df343\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b44366c-80d7-439e-998d-9fcc151df343\",\n \"name\": \"7b44366c-80d7-439e-998d-9fcc151df343\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:36.99482192Z\",\n \"endTime\": \"2026-10-09T19:09:45.381090093Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x199bbb710720\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-mrtmndgzlmdf\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b44366c-80d7-439e-998d-9fcc151df343\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b44366c-80d7-439e-998d-9fcc151df343\",\n \"name\": \"7b44366c-80d7-439e-998d-9fcc151df343\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:36.99482192Z\",\n \"endTime\": \"2026-10-09T19:09:45.381090093Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"30e91ea5a3f24c1968e6ad0a30fac6c72dd13f643a6e8e24a61a4e1c5c655706","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"605577627fca4f1f1e9afdeea753a9b0f1014b2292b57c1e35fc2434b6a85524","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xb3d17b32c20\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vzwh878hqpng\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b9f32dc-673d-478c-88d7-fc5808ea056e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b9f32dc-673d-478c-88d7-fc5808ea056e\",\n \"name\": \"4b9f32dc-673d-478c-88d7-fc5808ea056e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:50.079224939Z\",\n \"endTime\": \"2026-10-09T18:32:57.767672257Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xb3d17b32c20\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vzwh878hqpng\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b9f32dc-673d-478c-88d7-fc5808ea056e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4b9f32dc-673d-478c-88d7-fc5808ea056e\",\n \"name\": \"4b9f32dc-673d-478c-88d7-fc5808ea056e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:50.079224939Z\",\n \"endTime\": \"2026-10-09T18:32:57.767672257Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"35b0d1c1ddb2fa56f1a4a99f599a95e979d8780dd982a4fe8443eae0d16e97a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"7e936789cef771ddd9316ff31b81612b15375d9f6e0b7e2d85d82bd80d1f4752","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-59rb7vv6whnw/cluster-candidate-4-23-gk696c should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x37c98825a960\u003e: \n failed to create HCP cluster cluster-candidate-4-23-gk696c: failed waiting for cluster=\"cluster-candidate-4-23-gk696c\" in resourcegroup=\"rg-candidate-4-23-59rb7vv6whnw\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeb42bf-90f0-441c-a63e-aff4f3ce70ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeb42bf-90f0-441c-a63e-aff4f3ce70ff\",\n \"name\": \"afeb42bf-90f0-441c-a63e-aff4f3ce70ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:04.181877373Z\",\n \"endTime\": \"2026-10-09T18:31:07.60472581Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-59rb7vv6whnw/cluster-candidate-4-23-gk696c should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x37c98825a960\u003e: \n failed to create HCP cluster cluster-candidate-4-23-gk696c: failed waiting for cluster=\"cluster-candidate-4-23-gk696c\" in resourcegroup=\"rg-candidate-4-23-59rb7vv6whnw\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeb42bf-90f0-441c-a63e-aff4f3ce70ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeb42bf-90f0-441c-a63e-aff4f3ce70ff\",\n \"name\": \"afeb42bf-90f0-441c-a63e-aff4f3ce70ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:04.181877373Z\",\n \"endTime\": \"2026-10-09T18:31:07.60472581Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"378fb0f0c3138a7361fb01d5b61f57ed04c064dea4877b121da83cdc04d7f8f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"270f52452b182f15f724ad102109f03fa2af29e0483c091f90e700cd80a62b63","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:432]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0x32e4652764e0\u003e: \n failed waiting for cluster=\"acr-pull\" in resourcegroup=\"acr-pull-ghsrnfbzbs7w\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01cc1b85-cb04-4948-81af-a84ee1172f5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01cc1b85-cb04-4948-81af-a84ee1172f5e\",\n \"name\": \"01cc1b85-cb04-4948-81af-a84ee1172f5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:04.712611439Z\",\n \"endTime\": \"2026-10-09T18:32:07.994233939Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:432]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0x32e4652764e0\u003e: \n failed waiting for cluster=\"acr-pull\" in resourcegroup=\"acr-pull-ghsrnfbzbs7w\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01cc1b85-cb04-4948-81af-a84ee1172f5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/01cc1b85-cb04-4948-81af-a84ee1172f5e\",\n \"name\": \"01cc1b85-cb04-4948-81af-a84ee1172f5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:04.712611439Z\",\n \"endTime\": \"2026-10-09T18:32:07.994233939Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3961f5bad20f41aa11be7e97967ce5857fc5ab6bdf9895a13a0f7203357a86a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"5326b37b127a327cc885a7d06d11d2e5d1ae127ff662425651996afa5995d4c8","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x202a20e30bc0\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-vb2b7x9n2fn6\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d15c2d04-d306-443f-a667-d1fa4c91ef63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d15c2d04-d306-443f-a667-d1fa4c91ef63\",\n \"name\": \"d15c2d04-d306-443f-a667-d1fa4c91ef63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:58.922056002Z\",\n \"endTime\": \"2026-10-09T18:32:07.653066445Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x202a20e30bc0\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-vb2b7x9n2fn6\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d15c2d04-d306-443f-a667-d1fa4c91ef63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d15c2d04-d306-443f-a667-d1fa4c91ef63\",\n \"name\": \"d15c2d04-d306-443f-a667-d1fa4c91ef63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:58.922056002Z\",\n \"endTime\": \"2026-10-09T18:32:07.653066445Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3e0b47982e1a9e0d6730a91d3a3f4bbee22908572a087e247503a6ee08320087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"9ede499c304c2151498d5fa3f541dd8841118ad3c2e27507275b36b37bd4fa10","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3185feba73a0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-v28bzj42qplb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a41e5fcd-6e53-4c51-a232-f6e978e0b6f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a41e5fcd-6e53-4c51-a232-f6e978e0b6f4\",\n \"name\": \"a41e5fcd-6e53-4c51-a232-f6e978e0b6f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:02.593702135Z\",\n \"endTime\": \"2026-10-09T19:10:05.43294293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3185feba73a0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-v28bzj42qplb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a41e5fcd-6e53-4c51-a232-f6e978e0b6f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a41e5fcd-6e53-4c51-a232-f6e978e0b6f4\",\n \"name\": \"a41e5fcd-6e53-4c51-a232-f6e978e0b6f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:02.593702135Z\",\n \"endTime\": \"2026-10-09T19:10:05.43294293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"43db823a5a013c6af5c1d43fe52b9dd2f756da4c6315eddce993c246a27a2ef8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"86d0639f8a612df296007f755fdbe489736d2e4139e62ca248d9792696071e0c","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--zjz7xzqgvzph/cluster-hs-pre-6qmm4z should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x54f1cef8360\u003e: \n failed to create HCP cluster cluster-hs-pre-6qmm4z: failed waiting for cluster=\"cluster-hs-pre-6qmm4z\" in resourcegroup=\"rg-hs-pre--zjz7xzqgvzph\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13b41411-132a-4a05-bba1-afb761fbeedc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13b41411-132a-4a05-bba1-afb761fbeedc\",\n \"name\": \"13b41411-132a-4a05-bba1-afb761fbeedc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:26.932428728Z\",\n \"endTime\": \"2026-10-09T18:31:27.824979904Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--zjz7xzqgvzph/cluster-hs-pre-6qmm4z should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x54f1cef8360\u003e: \n failed to create HCP cluster cluster-hs-pre-6qmm4z: failed waiting for cluster=\"cluster-hs-pre-6qmm4z\" in resourcegroup=\"rg-hs-pre--zjz7xzqgvzph\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13b41411-132a-4a05-bba1-afb761fbeedc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13b41411-132a-4a05-bba1-afb761fbeedc\",\n \"name\": \"13b41411-132a-4a05-bba1-afb761fbeedc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:26.932428728Z\",\n \"endTime\": \"2026-10-09T18:31:27.824979904Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"45e0476d9fe478734034a3f89cda70c44d0033502ce46b2cc7c00b16cd45c9d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"b45da6a81fea23d6421698db5bd755ac99c4569b5c386a8df2278b531b67b4fd","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x177e38ef13a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-lsb6swwskkg2\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1669f29d-d6df-4267-9529-a0d7da468336\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1669f29d-d6df-4267-9529-a0d7da468336\",\n \"name\": \"1669f29d-d6df-4267-9529-a0d7da468336\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:51.468582358Z\",\n \"endTime\": \"2026-10-09T18:32:57.974613483Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x177e38ef13a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-lsb6swwskkg2\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1669f29d-d6df-4267-9529-a0d7da468336\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1669f29d-d6df-4267-9529-a0d7da468336\",\n \"name\": \"1669f29d-d6df-4267-9529-a0d7da468336\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:51.468582358Z\",\n \"endTime\": \"2026-10-09T18:32:57.974613483Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"47b4777164eb8606850254d601b67571acb4ba9822786f8bfcc9b0ad7f0c1717","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"eff3312fe070323cd5940b3c5e218a49e22dee4584f1fa0e3685bd035e905815","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e47d8a5e100\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-g4z4dttdrrdz\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/151550cf-073c-4dad-a405-aed2b7653edc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/151550cf-073c-4dad-a405-aed2b7653edc\",\n \"name\": \"151550cf-073c-4dad-a405-aed2b7653edc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:24.084781048Z\",\n \"endTime\": \"2026-10-09T18:31:27.863195526Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e47d8a5e100\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-g4z4dttdrrdz\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/151550cf-073c-4dad-a405-aed2b7653edc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/151550cf-073c-4dad-a405-aed2b7653edc\",\n \"name\": \"151550cf-073c-4dad-a405-aed2b7653edc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:24.084781048Z\",\n \"endTime\": \"2026-10-09T18:31:27.863195526Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4cc8baf3985353a37aee3505e010c784871e827bddd5bef411b78351e2dc1247","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"5fc73119cc4809fd8d58b3d22fa26a7089966e8c78623d913341e28668630c4d","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x79ffe50a2a0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-cxp698kznkfb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46fe2269-499f-4834-8ab6-b664af33b5a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46fe2269-499f-4834-8ab6-b664af33b5a2\",\n \"name\": \"46fe2269-499f-4834-8ab6-b664af33b5a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:08.122092327Z\",\n \"endTime\": \"2026-10-09T18:32:17.802040827Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x79ffe50a2a0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-cxp698kznkfb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46fe2269-499f-4834-8ab6-b664af33b5a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46fe2269-499f-4834-8ab6-b664af33b5a2\",\n \"name\": \"46fe2269-499f-4834-8ab6-b664af33b5a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:08.122092327Z\",\n \"endTime\": \"2026-10-09T18:32:17.802040827Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4e409817f28428beec1ba886d52bbbbd817ea7074c5fa038c7cba519e579290e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"929e070060761e0145c7786b83b10bf8854e5806952eb8de75267c6508fd7347","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x35e89de92dc0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-ncqj527js7qc\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6e242d6b-6cd0-441f-9e9f-ab6a2a8cecb1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6e242d6b-6cd0-441f-9e9f-ab6a2a8cecb1\",\n \"name\": \"6e242d6b-6cd0-441f-9e9f-ab6a2a8cecb1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:13.055503632Z\",\n \"endTime\": \"2026-10-09T18:31:18.192009212Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x35e89de92dc0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-ncqj527js7qc\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6e242d6b-6cd0-441f-9e9f-ab6a2a8cecb1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6e242d6b-6cd0-441f-9e9f-ab6a2a8cecb1\",\n \"name\": \"6e242d6b-6cd0-441f-9e9f-ab6a2a8cecb1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:13.055503632Z\",\n \"endTime\": \"2026-10-09T18:31:18.192009212Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4e93d9650e49cbe5e024b8d12da5aa469bdce19bf7f17f14c05e9c69887f39c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"471888eab8c388e835c310bd537c7afbb02ea604ccd87bd17c259c2e24db7e12","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-hzq568\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x39d60de28c00\u003e: \n failed to create HCP cluster cluster-zstream-4-23-hzq568: failed waiting for cluster=\"cluster-zstream-4-23-hzq568\" in resourcegroup=\"rg-zstream-upgrade-4-23-p2mrvhqrkcwk\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9025adf3-6286-4949-a9de-5c10722e6264\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9025adf3-6286-4949-a9de-5c10722e6264\",\n \"name\": \"9025adf3-6286-4949-a9de-5c10722e6264\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:19.664085411Z\",\n \"endTime\": \"2026-10-09T18:31:27.58037293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-hzq568\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x39d60de28c00\u003e: \n failed to create HCP cluster cluster-zstream-4-23-hzq568: failed waiting for cluster=\"cluster-zstream-4-23-hzq568\" in resourcegroup=\"rg-zstream-upgrade-4-23-p2mrvhqrkcwk\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9025adf3-6286-4949-a9de-5c10722e6264\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9025adf3-6286-4949-a9de-5c10722e6264\",\n \"name\": \"9025adf3-6286-4949-a9de-5c10722e6264\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:19.664085411Z\",\n \"endTime\": \"2026-10-09T18:31:27.58037293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"511c7029dac919703e4804bf19bb2630c92f6502ce7995bc46bd7ffd9dd8305b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"5d43281f44963b75ff8872a084fd7611e5d520a8395c99ac66916a5f44309d46","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-b925qf4dtv4p/cluster-candidate-5-1-9pxcgq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2eef7967c2e0\u003e: \n failed to create HCP cluster cluster-candidate-5-1-9pxcgq: failed waiting for cluster=\"cluster-candidate-5-1-9pxcgq\" in resourcegroup=\"rg-candidate-5-1-b925qf4dtv4p\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4fb4fa1d-4d37-4e68-9855-a3ff27e37126\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4fb4fa1d-4d37-4e68-9855-a3ff27e37126\",\n \"name\": \"4fb4fa1d-4d37-4e68-9855-a3ff27e37126\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:31.454683954Z\",\n \"endTime\": \"2026-10-09T18:31:38.135835036Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-b925qf4dtv4p/cluster-candidate-5-1-9pxcgq should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2eef7967c2e0\u003e: \n failed to create HCP cluster cluster-candidate-5-1-9pxcgq: failed waiting for cluster=\"cluster-candidate-5-1-9pxcgq\" in resourcegroup=\"rg-candidate-5-1-b925qf4dtv4p\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4fb4fa1d-4d37-4e68-9855-a3ff27e37126\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4fb4fa1d-4d37-4e68-9855-a3ff27e37126\",\n \"name\": \"4fb4fa1d-4d37-4e68-9855-a3ff27e37126\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:31.454683954Z\",\n \"endTime\": \"2026-10-09T18:31:38.135835036Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"52ee06d2224cf025b07d86e895c5b5bbc6df4ff54e49a3636ff313c6b027a795","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"0780aa479fe1109f7c3c28191facf899b2cfa00862be1a670b44484d844ab5f2","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-tq5mmp4dz9tr/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-tq5mmp4dz9tr/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-8jfp6hmxz2vs/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"552729dbeb06414888f839bcb98e4edc5f337e5e010ba33200566455ba9dead7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"36bc2df6248e6d14334a51d93f771ce0e45399416a0e41167b90f40cac1dccd6","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:421]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x335de76525e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-c656jk8nbqf7\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95424a21-7234-4d2f-b6f8-49db4fb067c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95424a21-7234-4d2f-b6f8-49db4fb067c9\",\n \"name\": \"95424a21-7234-4d2f-b6f8-49db4fb067c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:19.968557469Z\",\n \"endTime\": \"2026-10-09T18:31:27.896075751Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:421]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x335de76525e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-c656jk8nbqf7\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95424a21-7234-4d2f-b6f8-49db4fb067c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95424a21-7234-4d2f-b6f8-49db4fb067c9\",\n \"name\": \"95424a21-7234-4d2f-b6f8-49db4fb067c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:19.968557469Z\",\n \"endTime\": \"2026-10-09T18:31:27.896075751Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5ba0c2976b652eca0ca4d319a8ef719e5aa2f2602f633c64ba240c80aeb43f04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"01c7c9aba88a6bec22099f9efa23436203147ffbb00db28596435d46a08f8a58","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7c60a347e0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-wh6shv999rvk\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d322d275-9941-45bf-a054-aa9d4b20f7cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d322d275-9941-45bf-a054-aa9d4b20f7cc\",\n \"name\": \"d322d275-9941-45bf-a054-aa9d4b20f7cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.15141187Z\",\n \"endTime\": \"2026-10-09T18:31:17.847497361Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7c60a347e0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-wh6shv999rvk\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d322d275-9941-45bf-a054-aa9d4b20f7cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d322d275-9941-45bf-a054-aa9d4b20f7cc\",\n \"name\": \"d322d275-9941-45bf-a054-aa9d4b20f7cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.15141187Z\",\n \"endTime\": \"2026-10-09T18:31:17.847497361Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5cffff8db7f1c9d518d52ffdf882b5862ea8933754b466cc66394e7233aaba2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"6b365936ba6c5dc6e4955ca7b030bb08117bd6ac40551608167f9a3365250583","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x23c23ea44480\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-cxkvb78fvbzb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6df5258-6815-41f2-b102-078ded35b588\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6df5258-6815-41f2-b102-078ded35b588\",\n \"name\": \"d6df5258-6815-41f2-b102-078ded35b588\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:31.8726861Z\",\n \"endTime\": \"2026-10-09T18:31:37.732392148Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x23c23ea44480\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-cxkvb78fvbzb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6df5258-6815-41f2-b102-078ded35b588\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6df5258-6815-41f2-b102-078ded35b588\",\n \"name\": \"d6df5258-6815-41f2-b102-078ded35b588\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:31.8726861Z\",\n \"endTime\": \"2026-10-09T18:31:37.732392148Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5e9ae7d15513b9b1be7ebae2f1d8dd52da7aec9ae4325e5951bee6b90bb95dd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"648b2e6e6b01fe5aab9f8c1ce0cff5dbcd59c3427b11e6628df8162058113cea","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-4-22-27qv8b\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d26d7338bc0\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-4-22-27qv8b: failed waiting for cluster=\"cp-ystream-deny-pods-4-22-27qv8b\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-22-w7hdxl28d99s\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2e8a3a4-fc97-4e60-8ea1-fe9527661b48\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2e8a3a4-fc97-4e60-8ea1-fe9527661b48\",\n \"name\": \"f2e8a3a4-fc97-4e60-8ea1-fe9527661b48\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:58.253531041Z\",\n \"endTime\": \"2026-10-09T18:32:07.93537792Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-4-22-27qv8b\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d26d7338bc0\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-4-22-27qv8b: failed waiting for cluster=\"cp-ystream-deny-pods-4-22-27qv8b\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-22-w7hdxl28d99s\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2e8a3a4-fc97-4e60-8ea1-fe9527661b48\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2e8a3a4-fc97-4e60-8ea1-fe9527661b48\",\n \"name\": \"f2e8a3a4-fc97-4e60-8ea1-fe9527661b48\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:58.253531041Z\",\n \"endTime\": \"2026-10-09T18:32:07.93537792Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"622ce2d1a58debbb464c4e084104ce2d6b34391b79d4187d2593db37560fd0a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"9522ddba404ef96e63c679a657e30578373adb9cf594a75efc0375d22fc4ed5c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-sjswpn\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x2cc77affe3a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-sjswpn: failed waiting for cluster=\"cp-ystream-upgrade-5-0-sjswpn\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-b6ql69rn2p27\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c74b61a-0ab6-46a0-8dd1-69911db4a507\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c74b61a-0ab6-46a0-8dd1-69911db4a507\",\n \"name\": \"4c74b61a-0ab6-46a0-8dd1-69911db4a507\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:28.035516007Z\",\n \"endTime\": \"2026-10-09T18:31:37.653027995Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-sjswpn\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x2cc77affe3a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-sjswpn: failed waiting for cluster=\"cp-ystream-upgrade-5-0-sjswpn\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-b6ql69rn2p27\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c74b61a-0ab6-46a0-8dd1-69911db4a507\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4c74b61a-0ab6-46a0-8dd1-69911db4a507\",\n \"name\": \"4c74b61a-0ab6-46a0-8dd1-69911db4a507\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:28.035516007Z\",\n \"endTime\": \"2026-10-09T18:31:37.653027995Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6427162c2f7cf7484eadcb70f172f885db18a1c7a676bfa933886447fd9dea24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"6acc3aa5ad77e5f9b1c9ed7b19fd706b537dc9966ff8a237c8745e71ed06cb17","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0a71da6e20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pckr4j464gtb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db6745bc-bc9e-4dc4-bb47-135c48031922\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db6745bc-bc9e-4dc4-bb47-135c48031922\",\n \"name\": \"db6745bc-bc9e-4dc4-bb47-135c48031922\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:20.359791772Z\",\n \"endTime\": \"2026-10-09T18:31:27.841147423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0a71da6e20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pckr4j464gtb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db6745bc-bc9e-4dc4-bb47-135c48031922\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db6745bc-bc9e-4dc4-bb47-135c48031922\",\n \"name\": \"db6745bc-bc9e-4dc4-bb47-135c48031922\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:20.359791772Z\",\n \"endTime\": \"2026-10-09T18:31:27.841147423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"64f8abc377c65c67b6d0891aa78e9400249d3f28b1a3b2463634cbe6aa19e17c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"8be7e936168bcef852e361261d8601b6e2bd9c2e2e5761763283bfb55577d5af","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x67005e0a1e0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-lwp6hlccqt4d\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc683353-bd36-4308-9731-9574c60f012a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc683353-bd36-4308-9731-9574c60f012a\",\n \"name\": \"cc683353-bd36-4308-9731-9574c60f012a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:01.339352229Z\",\n \"endTime\": \"2026-10-09T18:32:07.902911075Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x67005e0a1e0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-lwp6hlccqt4d\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc683353-bd36-4308-9731-9574c60f012a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc683353-bd36-4308-9731-9574c60f012a\",\n \"name\": \"cc683353-bd36-4308-9731-9574c60f012a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:01.339352229Z\",\n \"endTime\": \"2026-10-09T18:32:07.902911075Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7b4beddbdf811ba8961fa8e4ea39b4930f902f8b9732831c7a570ca5a0232c0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"ccbadab87d718693551dd8538c82af6ce15efeceeac75de00a658396f4727bcf","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x89590912720\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bgt54n958sgj\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8af78792-3346-4e6f-9171-3d4b46deee20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8af78792-3346-4e6f-9171-3d4b46deee20\",\n \"name\": \"8af78792-3346-4e6f-9171-3d4b46deee20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:43.998867234Z\",\n \"endTime\": \"2026-10-09T18:31:48.180518414Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x89590912720\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bgt54n958sgj\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8af78792-3346-4e6f-9171-3d4b46deee20\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8af78792-3346-4e6f-9171-3d4b46deee20\",\n \"name\": \"8af78792-3346-4e6f-9171-3d4b46deee20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:43.998867234Z\",\n \"endTime\": \"2026-10-09T18:31:48.180518414Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"809b72671ed52e7547e51ac4a113f6a285654b1358c5c8e2fa444539d9f362a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"d24434ab2966d55d5744a0237f93d89b9f15a01822488b010acc27aa17508832","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-g72ptm\nUnexpected error:\n \u003c*fmt.wrapError | 0xb7f60d7e220\u003e: \n failed to create HCP cluster np-noedge-g72ptm: failed waiting for cluster=\"np-noedge-g72ptm\" in resourcegroup=\"rg-np-noedge-g72ptm-5wstl2fwg4bc\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/216a2acd-d28d-4c98-b531-56113e46cd95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/216a2acd-d28d-4c98-b531-56113e46cd95\",\n \"name\": \"216a2acd-d28d-4c98-b531-56113e46cd95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:59.413528951Z\",\n \"endTime\": \"2026-10-09T19:11:05.371290712Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-g72ptm\nUnexpected error:\n \u003c*fmt.wrapError | 0xb7f60d7e220\u003e: \n failed to create HCP cluster np-noedge-g72ptm: failed waiting for cluster=\"np-noedge-g72ptm\" in resourcegroup=\"rg-np-noedge-g72ptm-5wstl2fwg4bc\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/216a2acd-d28d-4c98-b531-56113e46cd95\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/216a2acd-d28d-4c98-b531-56113e46cd95\",\n \"name\": \"216a2acd-d28d-4c98-b531-56113e46cd95\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:59.413528951Z\",\n \"endTime\": \"2026-10-09T19:11:05.371290712Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8ebcad6078dfbf69448a05251268606a71db4860685b58456a350dba63d64814","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"cafba5839e877254327e3080d52936c7a82ce89903ec574f9060c3dec19d8fcc","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-qwwhdd\" with version 4.22.16 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2771bf5ab60\u003e: \n failed to create HCP cluster cluster-zstream-4-22-qwwhdd: failed waiting for cluster=\"cluster-zstream-4-22-qwwhdd\" in resourcegroup=\"rg-zstream-upgrade-4-22-kd9shtt6sxgz\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33605275-6441-4d00-9992-c176dbc8fcf4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33605275-6441-4d00-9992-c176dbc8fcf4\",\n \"name\": \"33605275-6441-4d00-9992-c176dbc8fcf4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:09.774030716Z\",\n \"endTime\": \"2026-10-09T18:31:17.776541303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-qwwhdd\" with version 4.22.16 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2771bf5ab60\u003e: \n failed to create HCP cluster cluster-zstream-4-22-qwwhdd: failed waiting for cluster=\"cluster-zstream-4-22-qwwhdd\" in resourcegroup=\"rg-zstream-upgrade-4-22-kd9shtt6sxgz\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33605275-6441-4d00-9992-c176dbc8fcf4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33605275-6441-4d00-9992-c176dbc8fcf4\",\n \"name\": \"33605275-6441-4d00-9992-c176dbc8fcf4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:09.774030716Z\",\n \"endTime\": \"2026-10-09T18:31:17.776541303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"94276c7d8999b48293f07ea96c21c540f9717d211f0c4935bacb0a20d6661e3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"d41480da135c16a57e520650419cfa359a9141d8dfe06e4e1787793d738cc6fe","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-wb9qld with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x114d5955a980\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-wb9qld: failed waiting for cluster=\"np-version-upgrade-cluster-wb9qld\" in resourcegroup=\"rg-np-version-upgrade-wb9qld-8xxjfp5c7xrb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a141e576-778c-4ae0-9f8d-563b92ca8c88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a141e576-778c-4ae0-9f8d-563b92ca8c88\",\n \"name\": \"a141e576-778c-4ae0-9f8d-563b92ca8c88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:53.187837968Z\",\n \"endTime\": \"2026-10-09T19:09:55.459750099Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-wb9qld with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x114d5955a980\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-wb9qld: failed waiting for cluster=\"np-version-upgrade-cluster-wb9qld\" in resourcegroup=\"rg-np-version-upgrade-wb9qld-8xxjfp5c7xrb\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a141e576-778c-4ae0-9f8d-563b92ca8c88\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a141e576-778c-4ae0-9f8d-563b92ca8c88\",\n \"name\": \"a141e576-778c-4ae0-9f8d-563b92ca8c88\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:53.187837968Z\",\n \"endTime\": \"2026-10-09T19:09:55.459750099Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"96eb0f9b672977d2db0588a6daeffcb48bcb8f775b6f77a6ddf522c784400411","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"a51a5142b300cc25f83121d87531a6477ee39ee81964278209d741153cbef2ce","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e00a251d180\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-dg68b6wdbl95\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b29c3a76-7e4d-4688-ba55-484661042381\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b29c3a76-7e4d-4688-ba55-484661042381\",\n \"name\": \"b29c3a76-7e4d-4688-ba55-484661042381\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:43:00.581974003Z\",\n \"endTime\": \"2026-10-09T19:12:05.363558255Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e00a251d180\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-dg68b6wdbl95\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b29c3a76-7e4d-4688-ba55-484661042381\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b29c3a76-7e4d-4688-ba55-484661042381\",\n \"name\": \"b29c3a76-7e4d-4688-ba55-484661042381\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:43:00.581974003Z\",\n \"endTime\": \"2026-10-09T19:12:05.363558255Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"99098f2538f31e6d88f61d6ea1876e3b282437369b1b2336edc942d9765a9e0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"23541533d46875257de182f86b81eb800fae0069577baf283b2393e3ecd99d6d","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c5369c50620\u003e: \n failed to create HCP cluster np-downgrade-rkvf2f: failed waiting for cluster=\"np-downgrade-rkvf2f\" in resourcegroup=\"rg-np-downgrade-rkvf2f-662jssrrw87b\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11078c9c-4b3d-492b-b5dd-86a31643023e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11078c9c-4b3d-492b-b5dd-86a31643023e\",\n \"name\": \"11078c9c-4b3d-492b-b5dd-86a31643023e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:59.737756895Z\",\n \"endTime\": \"2026-10-09T19:11:05.391503232Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c5369c50620\u003e: \n failed to create HCP cluster np-downgrade-rkvf2f: failed waiting for cluster=\"np-downgrade-rkvf2f\" in resourcegroup=\"rg-np-downgrade-rkvf2f-662jssrrw87b\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11078c9c-4b3d-492b-b5dd-86a31643023e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11078c9c-4b3d-492b-b5dd-86a31643023e\",\n \"name\": \"11078c9c-4b3d-492b-b5dd-86a31643023e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:59.737756895Z\",\n \"endTime\": \"2026-10-09T19:11:05.391503232Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9ac8a511e25be9564419301f0292fc88d48915b89995d3658b8d51b7ae1b9733","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"344bce9d5bb6db55aa99b4d0dae42ea14b220a1bbda168e54c277877dd20c8ba","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-xqlqq2\" with version 5.0.0-rc.4 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x31e860010700\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xqlqq2: failed waiting for cluster=\"cluster-zstream-5-0-xqlqq2\" in resourcegroup=\"rg-zstream-upgrade-5-0-24jbbvhcs69f\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/563513d2-4d5e-41f8-8927-4cdda458a533\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/563513d2-4d5e-41f8-8927-4cdda458a533\",\n \"name\": \"563513d2-4d5e-41f8-8927-4cdda458a533\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:31.119167931Z\",\n \"endTime\": \"2026-10-09T18:31:38.07679177Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-xqlqq2\" with version 5.0.0-rc.4 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x31e860010700\u003e: \n failed to create HCP cluster cluster-zstream-5-0-xqlqq2: failed waiting for cluster=\"cluster-zstream-5-0-xqlqq2\" in resourcegroup=\"rg-zstream-upgrade-5-0-24jbbvhcs69f\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/563513d2-4d5e-41f8-8927-4cdda458a533\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/563513d2-4d5e-41f8-8927-4cdda458a533\",\n \"name\": \"563513d2-4d5e-41f8-8927-4cdda458a533\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:31.119167931Z\",\n \"endTime\": \"2026-10-09T18:31:38.07679177Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a1cf308962b22afb87dc19ae54a4a615ae99f65d3e6884da520ef956083eea24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"18cbcc3172a62339984a3d37ac2466dcb61cbf74717c9d34584498668e6449e7","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xe827a362640\u003e: \n failed to create HCP cluster np-dg-minor-9q2848: failed waiting for cluster=\"np-dg-minor-9q2848\" in resourcegroup=\"rg-np-dg-minor-9q2848-bfwtcf254248\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d9b8b1-6079-4b0c-afab-f3603094a5e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d9b8b1-6079-4b0c-afab-f3603094a5e5\",\n \"name\": \"86d9b8b1-6079-4b0c-afab-f3603094a5e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:59.2025741Z\",\n \"endTime\": \"2026-10-09T19:11:05.47797695Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xe827a362640\u003e: \n failed to create HCP cluster np-dg-minor-9q2848: failed waiting for cluster=\"np-dg-minor-9q2848\" in resourcegroup=\"rg-np-dg-minor-9q2848-bfwtcf254248\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d9b8b1-6079-4b0c-afab-f3603094a5e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/86d9b8b1-6079-4b0c-afab-f3603094a5e5\",\n \"name\": \"86d9b8b1-6079-4b0c-afab-f3603094a5e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:59.2025741Z\",\n \"endTime\": \"2026-10-09T19:11:05.47797695Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a6458d358cdc3af02dd191db3f4f471772bdacbdf063f42288ca90587ae73f13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"d8e1a5f9d52ad8aee666439da2a8fa3c13b37b36cd8b70e932dec4d49375ed45","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:144]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x30228660e7e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-j7mf9xmh7lv5\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c9c95f39-63cf-434b-99f8-aff685f1dce0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c9c95f39-63cf-434b-99f8-aff685f1dce0\",\n \"name\": \"c9c95f39-63cf-434b-99f8-aff685f1dce0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:22.972274217Z\",\n \"endTime\": \"2026-10-09T18:31:27.809908038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:144]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x30228660e7e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-j7mf9xmh7lv5\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c9c95f39-63cf-434b-99f8-aff685f1dce0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c9c95f39-63cf-434b-99f8-aff685f1dce0\",\n \"name\": \"c9c95f39-63cf-434b-99f8-aff685f1dce0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:22.972274217Z\",\n \"endTime\": \"2026-10-09T18:31:27.809908038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a74efc088972cfba2b920bc4bc684e8de89a71d583d67af3a1f4c5c42bc13124","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"8e8ca9f4e7208efb5d4591e5e48ad01e865e6837bb0c14b95fb6aaa7aa5f6c67","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x13eb0e2191a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bkn7fgqj44mp\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-bkn7fgqj44mp\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x13eb0e00ddf0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x13eb0e2191a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bkn7fgqj44mp\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-bkn7fgqj44mp\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x13eb0e00ddf0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bkn7fgqj44mp\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded"}},{"row_id":"b1aaecbdb516bf13fc3b49d2d4d30f997d0abd458c97fa01991fb4ad5727016d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"6bd0886fe9be28c4793f4cbd79418d8f356d5615ce582d8d0371e979a9c5f77e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-tq7rmml6kns7/cluster-candidate-4-21-m4tzmh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x16bbe4e5c040\u003e: \n failed to create HCP cluster cluster-candidate-4-21-m4tzmh: failed waiting for cluster=\"cluster-candidate-4-21-m4tzmh\" in resourcegroup=\"rg-candidate-4-21-tq7rmml6kns7\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6529b35-2e31-464a-9bd0-fb3d59b6eb58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6529b35-2e31-464a-9bd0-fb3d59b6eb58\",\n \"name\": \"d6529b35-2e31-464a-9bd0-fb3d59b6eb58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:19.926181985Z\",\n \"endTime\": \"2026-10-09T18:31:27.888156177Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-tq7rmml6kns7/cluster-candidate-4-21-m4tzmh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x16bbe4e5c040\u003e: \n failed to create HCP cluster cluster-candidate-4-21-m4tzmh: failed waiting for cluster=\"cluster-candidate-4-21-m4tzmh\" in resourcegroup=\"rg-candidate-4-21-tq7rmml6kns7\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6529b35-2e31-464a-9bd0-fb3d59b6eb58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d6529b35-2e31-464a-9bd0-fb3d59b6eb58\",\n \"name\": \"d6529b35-2e31-464a-9bd0-fb3d59b6eb58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:19.926181985Z\",\n \"endTime\": \"2026-10-09T18:31:27.888156177Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b661892542995f21c12895f2318b280ded88d1aaf63c3e0c8817b23883a5aa68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"3a3d2162b324369cc9d26692d4078318e79e4e939c2e0cd7569a424308ab189d","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1cc812df71e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dfnkz2xl6z58\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/695cdd08-958c-4e2f-983d-9e12b25bf415\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/695cdd08-958c-4e2f-983d-9e12b25bf415\",\n \"name\": \"695cdd08-958c-4e2f-983d-9e12b25bf415\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:38:35.022304297Z\",\n \"endTime\": \"2026-10-09T19:07:35.444195011Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1cc812df71e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dfnkz2xl6z58\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/695cdd08-958c-4e2f-983d-9e12b25bf415\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/695cdd08-958c-4e2f-983d-9e12b25bf415\",\n \"name\": \"695cdd08-958c-4e2f-983d-9e12b25bf415\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:38:35.022304297Z\",\n \"endTime\": \"2026-10-09T19:07:35.444195011Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b8711f3e730c80f65c0fce5685c5d69d9cb612185ae4bd508ac10f64853aedf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"alert","signature_id":"ceaff42bdfa032f28d1f49bd3b397444e672e2f501892eadd1f4e851d4b9d1c5","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T19:17:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1936896-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1936896/providers/microsoft.monitor/accounts/hcps-j1936896\", namespace=\"ocm-arohcpci01-2tbdsg0oi92tdeeesp9b3hjkkb21uj1g-i5m2d9v6b0l3v4q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frw78\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbdsg0oi92tdeeesp9b3hjkkb21uj1g-i5m2d9v6b0l3v4q/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T19:17:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1936896-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1936896/providers/microsoft.monitor/accounts/hcps-j1936896\", namespace=\"ocm-arohcpci01-2tbdsg0oi92tdeeesp9b3hjkkb21uj1g-i5m2d9v6b0l3v4q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frw78\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbdsg0oi92tdeeesp9b3hjkkb21uj1g-i5m2d9v6b0l3v4q/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T19:17:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1936896-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1936896/providers/microsoft.monitor/accounts/hcps-j1936896\", namespace=\"ocm-arohcpci01-2tbdsg0oi92tdeeesp9b3hjkkb21uj1g-i5m2d9v6b0l3v4q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frw78\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbdsg0oi92tdeeesp9b3hjkkb21uj1g-i5m2d9v6b0l3v4q/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-09T19:17:47Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1936896-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1936896/providers/microsoft.monitor/accounts/hcps-j1936896\", namespace=\"ocm-arohcpci01-2tbdsg0oi92tdeeesp9b3hjkkb21uj1g-i5m2d9v6b0l3v4q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frw78\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tbdsg0oi92tdeeesp9b3hjkkb21uj1g-i5m2d9v6b0l3v4q/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"bdfffb714f86e2c8184cf94cff83840e261e5a26ba04dc0c7432b664112b4de7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"4bdea9cdd7191534eb3f205fc41751072adef3d93c0e8cb4a9f543ddbddc8724","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-5-0-92mvl6\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x2def1bfb6400\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-5-0-92mvl6: failed waiting for cluster=\"cp-ystream-deny-pods-5-0-92mvl6\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-mbjsvxff6788\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a366c997-6f86-4a02-b79f-33520fc978fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a366c997-6f86-4a02-b79f-33520fc978fd\",\n \"name\": \"a366c997-6f86-4a02-b79f-33520fc978fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:55.520160493Z\",\n \"endTime\": \"2026-10-09T18:31:58.019865769Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-5-0-92mvl6\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x2def1bfb6400\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-5-0-92mvl6: failed waiting for cluster=\"cp-ystream-deny-pods-5-0-92mvl6\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-mbjsvxff6788\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a366c997-6f86-4a02-b79f-33520fc978fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a366c997-6f86-4a02-b79f-33520fc978fd\",\n \"name\": \"a366c997-6f86-4a02-b79f-33520fc978fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:55.520160493Z\",\n \"endTime\": \"2026-10-09T18:31:58.019865769Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c41afc45832fa988802123ec3d8219c84ad9dba6d722f1d7ab64b8d8f8412480","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"e63114a71a857978b52cbd146d9854d7e3d351cfde3aff02e044cadc2bdd2d58","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-gv78x8dzj7jg/cluster-candidate-4-22-cf6vkf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x8447b69e100\u003e: \n failed to create HCP cluster cluster-candidate-4-22-cf6vkf: failed waiting for cluster=\"cluster-candidate-4-22-cf6vkf\" in resourcegroup=\"rg-candidate-4-22-gv78x8dzj7jg\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d732952-987f-47ff-98c9-4b6fc716c2d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d732952-987f-47ff-98c9-4b6fc716c2d0\",\n \"name\": \"8d732952-987f-47ff-98c9-4b6fc716c2d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:30.381493598Z\",\n \"endTime\": \"2026-10-09T18:31:38.035050636Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-gv78x8dzj7jg/cluster-candidate-4-22-cf6vkf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x8447b69e100\u003e: \n failed to create HCP cluster cluster-candidate-4-22-cf6vkf: failed waiting for cluster=\"cluster-candidate-4-22-cf6vkf\" in resourcegroup=\"rg-candidate-4-22-gv78x8dzj7jg\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d732952-987f-47ff-98c9-4b6fc716c2d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d732952-987f-47ff-98c9-4b6fc716c2d0\",\n \"name\": \"8d732952-987f-47ff-98c9-4b6fc716c2d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:30.381493598Z\",\n \"endTime\": \"2026-10-09T18:31:38.035050636Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c63e0bd7c85413785b6a44f24c70ab2b858666605d6fffc1f71679773f2614a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"2febc8fb20e64b0398ded29dc23013a05f1468e452028bf82ed4f95823d3e067","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:98]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xee8e0b28e60\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-2lrhvqr2hpk6\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f458ec-25ac-4ad4-8d4f-bb9cf5274285\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f458ec-25ac-4ad4-8d4f-bb9cf5274285\",\n \"name\": \"24f458ec-25ac-4ad4-8d4f-bb9cf5274285\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:09.548074629Z\",\n \"endTime\": \"2026-10-09T18:32:17.843512898Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:98]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xee8e0b28e60\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-2lrhvqr2hpk6\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f458ec-25ac-4ad4-8d4f-bb9cf5274285\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24f458ec-25ac-4ad4-8d4f-bb9cf5274285\",\n \"name\": \"24f458ec-25ac-4ad4-8d4f-bb9cf5274285\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:09.548074629Z\",\n \"endTime\": \"2026-10-09T18:32:17.843512898Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"caba4adecb08d585dae9b6f9a90723ac1800716ef7d98524b60892ee23802deb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"95092931f0b18c876d556c51bf269aaa70f521731d5e8948e3036ee0f211a9b5","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3969761f4860\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-2wqwxnq2ff6g\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3905ee8d-705b-4fc0-9d67-f7496f5b640c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3905ee8d-705b-4fc0-9d67-f7496f5b640c\",\n \"name\": \"3905ee8d-705b-4fc0-9d67-f7496f5b640c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:29.879376585Z\",\n \"endTime\": \"2026-10-09T18:31:37.637229023Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3969761f4860\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-2wqwxnq2ff6g\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3905ee8d-705b-4fc0-9d67-f7496f5b640c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3905ee8d-705b-4fc0-9d67-f7496f5b640c\",\n \"name\": \"3905ee8d-705b-4fc0-9d67-f7496f5b640c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:29.879376585Z\",\n \"endTime\": \"2026-10-09T18:31:37.637229023Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cfcfd2e85ce9e273a896f775306808d9f3bd6081ed9dd0b3bd3562f96d158fde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"alert","signature_id":"833cf2b5be700efed721d7f0826a89d849bf5421fcf520d4fb7e22e13dcd387a","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T18:38:22Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j1936896-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1936896/providers/microsoft.monitor/accounts/services-j1936896\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of completed cluster create (install) operations on ci01-j1936896-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired"}},{"row_id":"d2653450002838b05ee276a1319f34d077bfc47062e408f04807f6b0e2467e23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"fe5361e6f948427f64311e91b3e4d028f213c3abcab12243ef538ed838350160","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x6e954516aa0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-ch4snv2mkn8l\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/963657bb-3c12-49eb-aad6-6ef799365ba8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/963657bb-3c12-49eb-aad6-6ef799365ba8\",\n \"name\": \"963657bb-3c12-49eb-aad6-6ef799365ba8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:42:48.756939438Z\",\n \"endTime\": \"2026-10-09T19:11:55.377380525Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x6e954516aa0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-ch4snv2mkn8l\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/963657bb-3c12-49eb-aad6-6ef799365ba8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/963657bb-3c12-49eb-aad6-6ef799365ba8\",\n \"name\": \"963657bb-3c12-49eb-aad6-6ef799365ba8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:42:48.756939438Z\",\n \"endTime\": \"2026-10-09T19:11:55.377380525Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d2e63efcf5c0127d34bb1649b4d99b1d7dfc12f60aafbc316be3209a28903b18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"aa8d5fde17c317b9352aca696c164c9e194b56fe7f66ffe5c727b348eda15428","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x7f2f57b95f0\u003e: \n failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hwcbq22lzw9l\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-hwcbq22lzw9l\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x7f2f606c4c0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l\",\n },\n \u003c*fmt.wrapErrors | 0x7f2f57b95c0\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-hwcbq22lzw9l\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x7f2f606c4c0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x7f2f57b95f0\u003e: \n failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hwcbq22lzw9l\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-hwcbq22lzw9l\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x7f2f606c4c0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l\",\n },\n \u003c*fmt.wrapErrors | 0x7f2f57b95c0\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-hwcbq22lzw9l\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x7f2f606c4c0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hwcbq22lzw9l\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-bkn7fgqj44mp\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-bkn7fgqj44mp, error: context deadline exceeded"}},{"row_id":"d43c1ef614d251a65168c7265eb7b705301f10761220be057a6dc5d18759f294","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"alert","signature_id":"df684fe7f4e6c9e63460594adb2966f6b38fe7c13181dc1da0d9f3bfdbb319bb","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T19:03:21Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j1936896-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1936896/providers/microsoft.monitor/accounts/services-j1936896\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of completed cluster create (install) operations on ci01-j1936896-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired"}},{"row_id":"d47f249735ecb84556d29e671582665842c87bce6163e2d31e575674d7b3bdd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"1a225498f788e362cfb0798ecc880ee9e0d615e421befffa8b4c39c1db09ea76","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-shl6mv\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc608374a6e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-shl6mv: failed waiting for cluster=\"cp-ystream-upgrade-4-23-shl6mv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-4cxlmv6b22n7\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4792cc94-7e2d-4032-bd4f-acb61864439e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4792cc94-7e2d-4032-bd4f-acb61864439e\",\n \"name\": \"4792cc94-7e2d-4032-bd4f-acb61864439e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:29.794897918Z\",\n \"endTime\": \"2026-10-09T18:31:38.092037296Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-shl6mv\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc608374a6e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-shl6mv: failed waiting for cluster=\"cp-ystream-upgrade-4-23-shl6mv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-4cxlmv6b22n7\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4792cc94-7e2d-4032-bd4f-acb61864439e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4792cc94-7e2d-4032-bd4f-acb61864439e\",\n \"name\": \"4792cc94-7e2d-4032-bd4f-acb61864439e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:29.794897918Z\",\n \"endTime\": \"2026-10-09T18:31:38.092037296Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d50bf5249b84d67a0a3e0bdfa2fc44b9cd77664bf95dc66f79d92ab22d7c6460","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"47c55e31574be527ea313db7769d85b3ccf1d32158e37662d651889d8dce39cf","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bdd1b9e300\u003e: \n failed to create HCP cluster ea-listjxcfch: failed waiting for cluster=\"ea-listjxcfch\" in resourcegroup=\"ea-list-bwj9cp8sqnd5\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c8059bfb-fd3f-4e24-b492-e6ec0c766279\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c8059bfb-fd3f-4e24-b492-e6ec0c766279\",\n \"name\": \"c8059bfb-fd3f-4e24-b492-e6ec0c766279\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:06.045432669Z\",\n \"endTime\": \"2026-10-09T18:32:07.780692137Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bdd1b9e300\u003e: \n failed to create HCP cluster ea-listjxcfch: failed waiting for cluster=\"ea-listjxcfch\" in resourcegroup=\"ea-list-bwj9cp8sqnd5\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c8059bfb-fd3f-4e24-b492-e6ec0c766279\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c8059bfb-fd3f-4e24-b492-e6ec0c766279\",\n \"name\": \"c8059bfb-fd3f-4e24-b492-e6ec0c766279\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:06.045432669Z\",\n \"endTime\": \"2026-10-09T18:32:07.780692137Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d5333f6e81d8d30dbfb1157a8e8768d5993268b0cd40021fdc2d67ca642fc7f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"cff19b5d202fc005cd43e8cdf242df5b3a41239c4346b5edd14d96775c35cc0e","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-4-23-kvt24g\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x273a184a5d60\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-4-23-kvt24g: failed waiting for cluster=\"cp-ystream-deny-pods-4-23-kvt24g\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-23-ftdlqcp8hmwk\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b70812c-ce11-40d5-b792-3d06ddb946ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b70812c-ce11-40d5-b792-3d06ddb946ed\",\n \"name\": \"6b70812c-ce11-40d5-b792-3d06ddb946ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:06.839137502Z\",\n \"endTime\": \"2026-10-09T18:32:07.991193612Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:161]: failed to create HCP cluster \"cp-ystream-deny-pods-4-23-kvt24g\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x273a184a5d60\u003e: \n failed to create HCP cluster cp-ystream-deny-pods-4-23-kvt24g: failed waiting for cluster=\"cp-ystream-deny-pods-4-23-kvt24g\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-23-ftdlqcp8hmwk\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b70812c-ce11-40d5-b792-3d06ddb946ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b70812c-ce11-40d5-b792-3d06ddb946ed\",\n \"name\": \"6b70812c-ce11-40d5-b792-3d06ddb946ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:06.839137502Z\",\n \"endTime\": \"2026-10-09T18:32:07.991193612Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e0f27e5b180db0b62a1d75e37d499e480cfd83314fd37e0d3951cd5a5bb84b4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"101829a1e4b673aa4d7ffb71716277ad9438dad959bbce3ad102021a727eb3fe","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f8cfca18160\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-d9vjk88cdxmv\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/715e9b71-ab21-4169-9e2e-f3f804cb80c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/715e9b71-ab21-4169-9e2e-f3f804cb80c0\",\n \"name\": \"715e9b71-ab21-4169-9e2e-f3f804cb80c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:40.105550226Z\",\n \"endTime\": \"2026-10-09T18:31:47.946839601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f8cfca18160\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-d9vjk88cdxmv\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/715e9b71-ab21-4169-9e2e-f3f804cb80c0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/715e9b71-ab21-4169-9e2e-f3f804cb80c0\",\n \"name\": \"715e9b71-ab21-4169-9e2e-f3f804cb80c0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:40.105550226Z\",\n \"endTime\": \"2026-10-09T18:31:47.946839601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e10bee9c5b50866ccd576af3a34c99899febf11c1c7ccbafad9fda6bdf27f2e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"aa7ef0a640cd2e5f193ec1087698a362ab938fd455bc16df8d04503eb0313d89","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-hkw56n\" with version 4.20.40 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x149ef4a0e680\u003e: \n failed to create HCP cluster cluster-zstream-4-20-hkw56n: failed waiting for cluster=\"cluster-zstream-4-20-hkw56n\" in resourcegroup=\"rg-zstream-upgrade-4-20-sp6t2zkkxgkt\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9add752-dc4d-4949-a0b4-888b3fea1164\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9add752-dc4d-4949-a0b4-888b3fea1164\",\n \"name\": \"e9add752-dc4d-4949-a0b4-888b3fea1164\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:27.811481675Z\",\n \"endTime\": \"2026-10-09T18:31:27.878574533Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-hkw56n\" with version 4.20.40 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x149ef4a0e680\u003e: \n failed to create HCP cluster cluster-zstream-4-20-hkw56n: failed waiting for cluster=\"cluster-zstream-4-20-hkw56n\" in resourcegroup=\"rg-zstream-upgrade-4-20-sp6t2zkkxgkt\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9add752-dc4d-4949-a0b4-888b3fea1164\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e9add752-dc4d-4949-a0b4-888b3fea1164\",\n \"name\": \"e9add752-dc4d-4949-a0b4-888b3fea1164\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:27.811481675Z\",\n \"endTime\": \"2026-10-09T18:31:27.878574533Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e1e95ac9b2a1739cd4e04e6c8813a078d58f2c8efeaff4bda0287a647210920e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"573c2968f03b5e1da4c26a98373bd1ec66da67dde8c5072bba353f3bec23cb8f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x301e62396440\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7jw7nqhsrrj\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dda75f8f-4faa-422a-ac16-0ae5a3cb1048\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dda75f8f-4faa-422a-ac16-0ae5a3cb1048\",\n \"name\": \"dda75f8f-4faa-422a-ac16-0ae5a3cb1048\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:44.098063409Z\",\n \"endTime\": \"2026-10-09T18:31:48.324258901Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0x301e62396440\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z7jw7nqhsrrj\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dda75f8f-4faa-422a-ac16-0ae5a3cb1048\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dda75f8f-4faa-422a-ac16-0ae5a3cb1048\",\n \"name\": \"dda75f8f-4faa-422a-ac16-0ae5a3cb1048\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:44.098063409Z\",\n \"endTime\": \"2026-10-09T18:31:48.324258901Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e31908546c67837f87f2ef53cc619997189b06742bd0bf53e49517ed93eab0a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"b7b1480a1ea53486737434398aa7a46d927a9d5857abd693d1a00c9d13054a37","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-r8nmpd with version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x2571f3e86580\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-r8nmpd: failed waiting for cluster=\"np-version-upgrade-cluster-r8nmpd\" in resourcegroup=\"rg-np-version-upgrade-r8nmpd-gs94bsxjkcq8\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3582a7c9-6838-4b54-983d-f11c1e0c0069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3582a7c9-6838-4b54-983d-f11c1e0c0069\",\n \"name\": \"3582a7c9-6838-4b54-983d-f11c1e0c0069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:58.482094136Z\",\n \"endTime\": \"2026-10-09T19:11:05.361143305Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-r8nmpd with version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x2571f3e86580\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-r8nmpd: failed waiting for cluster=\"np-version-upgrade-cluster-r8nmpd\" in resourcegroup=\"rg-np-version-upgrade-r8nmpd-gs94bsxjkcq8\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3582a7c9-6838-4b54-983d-f11c1e0c0069\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3582a7c9-6838-4b54-983d-f11c1e0c0069\",\n \"name\": \"3582a7c9-6838-4b54-983d-f11c1e0c0069\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:58.482094136Z\",\n \"endTime\": \"2026-10-09T19:11:05.361143305Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e56691e9da0513b69ce769c6f1984137ee3f39f0ce92615036f0ab7e31f0f0ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"3dd4cf855356a60f412d65e778d1d59b4a3d625c3e48837dad30fa890f5c5ea7","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-q98qg9\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x6049f4b8880\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-q98qg9: failed waiting for cluster=\"cp-ystream-upgrade-5-1-q98qg9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-wnb9rxjktgkl\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1c38232c-6d4d-40dc-b43c-a5f7246c3ad8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1c38232c-6d4d-40dc-b43c-a5f7246c3ad8\",\n \"name\": \"1c38232c-6d4d-40dc-b43c-a5f7246c3ad8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:29.626443008Z\",\n \"endTime\": \"2026-10-09T18:31:38.091422062Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-q98qg9\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x6049f4b8880\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-q98qg9: failed waiting for cluster=\"cp-ystream-upgrade-5-1-q98qg9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-wnb9rxjktgkl\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1c38232c-6d4d-40dc-b43c-a5f7246c3ad8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1c38232c-6d4d-40dc-b43c-a5f7246c3ad8\",\n \"name\": \"1c38232c-6d4d-40dc-b43c-a5f7246c3ad8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:29.626443008Z\",\n \"endTime\": \"2026-10-09T18:31:38.091422062Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e65dc07590d558dafb4a1b0b5641a22374db1f034f60cab34f39fcae7ca52dce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"c9a888ba363bac1a8695dd65eb2abf121932544cc844047c0c9256e6cecb2493","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-4ppqcm with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e64447d2540\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-4ppqcm: failed waiting for cluster=\"np-version-upgrade-cluster-4ppqcm\" in resourcegroup=\"rg-np-version-upgrade-4ppqcm-d4stdt7h7cd7\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2e73d1f6-3086-4b9b-9df2-64fd1cd60773\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2e73d1f6-3086-4b9b-9df2-64fd1cd60773\",\n \"name\": \"2e73d1f6-3086-4b9b-9df2-64fd1cd60773\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:53.127528563Z\",\n \"endTime\": \"2026-10-09T19:09:55.435737413Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-4ppqcm with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e64447d2540\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-4ppqcm: failed waiting for cluster=\"np-version-upgrade-cluster-4ppqcm\" in resourcegroup=\"rg-np-version-upgrade-4ppqcm-d4stdt7h7cd7\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2e73d1f6-3086-4b9b-9df2-64fd1cd60773\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2e73d1f6-3086-4b9b-9df2-64fd1cd60773\",\n \"name\": \"2e73d1f6-3086-4b9b-9df2-64fd1cd60773\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:40:53.127528563Z\",\n \"endTime\": \"2026-10-09T19:09:55.435737413Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f5a5ce7771ebd3a9e1d6207fbc377b53c4b7703d2290ec8f391f80173c9b0694","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"6c5019cbf95e4513f1c0325e65420dcf50b2b8b9b1bc354a5bc737088dcdb0dd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-4q2hm4p8flft/cluster-candidate-5-0-7p85mb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x29bfd3f10a60\u003e: \n failed to create HCP cluster cluster-candidate-5-0-7p85mb: failed waiting for cluster=\"cluster-candidate-5-0-7p85mb\" in resourcegroup=\"rg-candidate-5-0-4q2hm4p8flft\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf5d31f8-b2b3-4e7f-95ac-accad89f8a22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf5d31f8-b2b3-4e7f-95ac-accad89f8a22\",\n \"name\": \"cf5d31f8-b2b3-4e7f-95ac-accad89f8a22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.48921674Z\",\n \"endTime\": \"2026-10-09T18:31:17.829305884Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-4q2hm4p8flft/cluster-candidate-5-0-7p85mb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x29bfd3f10a60\u003e: \n failed to create HCP cluster cluster-candidate-5-0-7p85mb: failed waiting for cluster=\"cluster-candidate-5-0-7p85mb\" in resourcegroup=\"rg-candidate-5-0-4q2hm4p8flft\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf5d31f8-b2b3-4e7f-95ac-accad89f8a22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf5d31f8-b2b3-4e7f-95ac-accad89f8a22\",\n \"name\": \"cf5d31f8-b2b3-4e7f-95ac-accad89f8a22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:10.48921674Z\",\n \"endTime\": \"2026-10-09T18:31:17.829305884Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f7400c1c81205d8087da7935f55d5c28ff2fc8a84d1bc8a0c54fda12387ed9ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"9a69793d927870af610e2db8ebc5fb307ca450c095c5a0f6f70d0ba353868781","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-mz6c7q2wdqbz/cluster-candidate-4-20-5nddfd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x17af8efbc1e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-5nddfd: failed waiting for cluster=\"cluster-candidate-4-20-5nddfd\" in resourcegroup=\"rg-candidate-4-20-mz6c7q2wdqbz\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e696d5b3-02d4-4f4a-8473-f34742a9e396\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e696d5b3-02d4-4f4a-8473-f34742a9e396\",\n \"name\": \"e696d5b3-02d4-4f4a-8473-f34742a9e396\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:00.235189165Z\",\n \"endTime\": \"2026-10-09T18:32:07.930455779Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-mz6c7q2wdqbz/cluster-candidate-4-20-5nddfd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x17af8efbc1e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-5nddfd: failed waiting for cluster=\"cluster-candidate-4-20-5nddfd\" in resourcegroup=\"rg-candidate-4-20-mz6c7q2wdqbz\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e696d5b3-02d4-4f4a-8473-f34742a9e396\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e696d5b3-02d4-4f4a-8473-f34742a9e396\",\n \"name\": \"e696d5b3-02d4-4f4a-8473-f34742a9e396\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:03:00.235189165Z\",\n \"endTime\": \"2026-10-09T18:32:07.930455779Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f8def5375250ba236bbc00b06b6a5d443696f1d83ec2e9965ff5779b574e2d3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"64a208342c453fb0e3e24af80d2b52ba48d347f00960fd0fb86861c86009e388","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-v4mqdl at version 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0x39d668b22540\u003e: \n failed to create HCP cluster np-skip-minor-v4mqdl: failed waiting for cluster=\"np-skip-minor-v4mqdl\" in resourcegroup=\"rg-np-skip-minor-v4mqdl-mjk8skltxp9z\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88d993e2-c096-42e9-a517-e34a00ecb286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88d993e2-c096-42e9-a517-e34a00ecb286\",\n \"name\": \"88d993e2-c096-42e9-a517-e34a00ecb286\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:59.152989858Z\",\n \"endTime\": \"2026-10-09T19:11:05.472167684Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-v4mqdl at version 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0x39d668b22540\u003e: \n failed to create HCP cluster np-skip-minor-v4mqdl: failed waiting for cluster=\"np-skip-minor-v4mqdl\" in resourcegroup=\"rg-np-skip-minor-v4mqdl-mjk8skltxp9z\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88d993e2-c096-42e9-a517-e34a00ecb286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/88d993e2-c096-42e9-a517-e34a00ecb286\",\n \"name\": \"88d993e2-c096-42e9-a517-e34a00ecb286\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:41:59.152989858Z\",\n \"endTime\": \"2026-10-09T19:11:05.472167684Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"faaa2dcf41f7269061662fee7fcd0ade58d94c610ab47140d6ba3b1e3b3a568e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"42e03a759fcb36e1ff575851b5b1f1da7f4bd1ebde306828e827dc02e2199766","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-4sfvrk\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x3edb50de6460\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-4sfvrk: failed waiting for cluster=\"cp-ystream-upgrade-4-22-4sfvrk\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-2msl55xkftls\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e56901b2-67f6-4317-a7ca-ba4d115c9a3b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e56901b2-67f6-4317-a7ca-ba4d115c9a3b\",\n \"name\": \"e56901b2-67f6-4317-a7ca-ba4d115c9a3b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:17.464807841Z\",\n \"endTime\": \"2026-10-09T18:31:17.868854868Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-4sfvrk\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0x3edb50de6460\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-4sfvrk: failed waiting for cluster=\"cp-ystream-upgrade-4-22-4sfvrk\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-2msl55xkftls\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e56901b2-67f6-4317-a7ca-ba4d115c9a3b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e56901b2-67f6-4317-a7ca-ba4d115c9a3b\",\n \"name\": \"e56901b2-67f6-4317-a7ca-ba4d115c9a3b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:17.464807841Z\",\n \"endTime\": \"2026-10-09T18:31:17.868854868Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ffcf9c19f27bc40bafcc8cb6c58a3f49227f0186a1508a72c657319c715ae346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607415241936896","occurred_at":"2026-10-09T17:15:54Z","failed_at":"e2e","signature_id":"7834d516ec38d9112f9510dec32a0654c9a7a541797540701f62a268ef5c8e78","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-79n8sz\" with version 4.21.35 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2331d37ea500\u003e: \n failed to create HCP cluster cluster-zstream-4-21-79n8sz: failed waiting for cluster=\"cluster-zstream-4-21-79n8sz\" in resourcegroup=\"rg-zstream-upgrade-4-21-dc4tt6ftvkcp\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70dc33e8-b20d-4994-a031-48efd910a40d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70dc33e8-b20d-4994-a031-48efd910a40d\",\n \"name\": \"70dc33e8-b20d-4994-a031-48efd910a40d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:09.770843031Z\",\n \"endTime\": \"2026-10-09T18:31:17.79586721Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-79n8sz\" with version 4.21.35 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2331d37ea500\u003e: \n failed to create HCP cluster cluster-zstream-4-21-79n8sz: failed waiting for cluster=\"cluster-zstream-4-21-79n8sz\" in resourcegroup=\"rg-zstream-upgrade-4-21-dc4tt6ftvkcp\" to finish creating: GET https://rp.j1936896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70dc33e8-b20d-4994-a031-48efd910a40d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70dc33e8-b20d-4994-a031-48efd910a40d\",\n \"name\": \"70dc33e8-b20d-4994-a031-48efd910a40d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T18:02:09.770843031Z\",\n \"endTime\": \"2026-10-09T18:31:17.79586721Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [defaultIngressServingCertificate] default ingress certificate not yet synced; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":73,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","d59dd78da58331fe4e550f023f7e8f060829ae2cbeeb3978410f21452c0b34b8","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":73,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607106348224512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"closed","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T17:14:44Z","started_at":"2026-10-09T17:14:44Z","completed_at":"2026-10-09T19:33:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T19:46:54.472587977Z","timing_metadata_checked_at":"2026-10-09T19:46:54.472587977Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T19:46:54.476082567Z","region_metadata_checked_at":"2026-10-09T19:46:54.476082567Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"50daec2edb3fa7ddd6e06e3a8033583be5ba31f73b0787a67fbf979f6c618f9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108607106348224512","occurred_at":"2026-10-09T17:14:44Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108594476522934272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"open","pr_sha":"9cb1004274b727280d33f5a8712c40a5ce992541","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T16:24:33Z","started_at":"2026-10-09T16:24:33Z","completed_at":"2026-10-09T17:06:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T17:14:49.953593691Z","timing_metadata_checked_at":"2026-10-09T17:14:49.953593691Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T17:14:49.956978181Z","region_metadata_checked_at":"2026-10-09T17:14:49.956978181Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"472fbace177eb1ae88d8422b259d34124f48181ca3540a4f6c7fae18a35286dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108594476522934272","occurred_at":"2026-10-09T16:24:33Z","failed_at":"provision","signature_id":"3eea31dd2ba80ae3c26eda9638a5c4961c637f75b7b51fe4771e43332561b111","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-09T16:49:10.013Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-09T16:49:10.529Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-09T16:49:10.529Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-09T16:49:11.005Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 namespace=aro-hcp-mitigations\ntime=2026-10-09T16:49:11.316Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 dryRun=false\ntime=2026-10-09T16:49:11.316Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-09T16:49:11.444Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1\ntime=2026-10-09T16:55:10.359Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-09T16:55:10.360Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108589017825021952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7227,"pr_state":"closed","pr_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","final_merged_sha":"349d3b8cc97ab39f5e30ce2c85d07daf9bb7411c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T16:02:30Z","started_at":"2026-10-09T16:02:30Z","completed_at":"2026-10-09T18:33:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T18:34:39.279377443Z","timing_metadata_checked_at":"2026-10-09T18:34:39.279377443Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T18:34:39.283797729Z","region_metadata_checked_at":"2026-10-09T18:34:39.283797729Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9aa1298057879549743c2d7809b9db33eef1eb128a5adbfe7a9a5a7696244f50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108589017825021952","occurred_at":"2026-10-09T16:02:30Z","failed_at":"e2e","signature_id":"7ac519174700b8af8b38e72050ab85056e2459914e3de2dee5a59bdd798501b7","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-2xlsbr with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x2653bb4a2240\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-2xlsbr: failed waiting for cluster=\"np-version-upgrade-cluster-2xlsbr\" in resourcegroup=\"rg-np-version-upgrade-2xlsbr-hc74hngnvw7h\" to finish creating: GET https://rp.j5021952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9aa7f9a2-8731-49d2-9f2d-e0d6dc9de68c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9aa7f9a2-8731-49d2-9f2d-e0d6dc9de68c\",\n \"name\": \"9aa7f9a2-8731-49d2-9f2d-e0d6dc9de68c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T17:24:44.640587758Z\",\n \"endTime\": \"2026-10-09T17:53:46.503674235Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-2xlsbr with version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x2653bb4a2240\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-2xlsbr: failed waiting for cluster=\"np-version-upgrade-cluster-2xlsbr\" in resourcegroup=\"rg-np-version-upgrade-2xlsbr-hc74hngnvw7h\" to finish creating: GET https://rp.j5021952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9aa7f9a2-8731-49d2-9f2d-e0d6dc9de68c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9aa7f9a2-8731-49d2-9f2d-e0d6dc9de68c\",\n \"name\": \"9aa7f9a2-8731-49d2-9f2d-e0d6dc9de68c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T17:24:44.640587758Z\",\n \"endTime\": \"2026-10-09T17:53:46.503674235Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108576502927855616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7437,"pr_state":"closed","pr_sha":"7495c811b9a906fee2419ebd6d9df2930140dd35","final_merged_sha":"7495c811b9a906fee2419ebd6d9df2930140dd35","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T15:13:08Z","started_at":"2026-10-09T15:13:08Z","completed_at":"2026-10-09T17:27:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T17:36:06.88514989Z","timing_metadata_checked_at":"2026-10-09T17:36:06.88514989Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T17:36:06.888013981Z","region_metadata_checked_at":"2026-10-09T17:36:06.888013981Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108571016249741312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7305,"pr_state":"open","pr_sha":"6210b7bdb19602dd76919df4a74af1eac0dabffe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T14:51:20Z","started_at":"2026-10-09T14:51:20Z","completed_at":"2026-10-09T17:20:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T17:25:08.80678378Z","timing_metadata_checked_at":"2026-10-09T17:25:08.80678378Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T17:25:08.812165963Z","region_metadata_checked_at":"2026-10-09T17:25:08.812165963Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108569711095582720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7444,"pr_state":"open","pr_sha":"4efbf6bf8baa7e1355ea06418637c2c256e8ccab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T14:46:09Z","started_at":"2026-10-09T14:46:09Z","completed_at":"2026-10-09T17:21:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T17:25:09.023432786Z","timing_metadata_checked_at":"2026-10-09T17:25:09.023432786Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T17:25:09.026185577Z","region_metadata_checked_at":"2026-10-09T17:25:09.026185577Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"44fbec62626fe9fa74809c36e5a1f35c5dba0e3622f454deea1fe0cdd643b2bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7444/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108569711095582720","occurred_at":"2026-10-09T14:46:09Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-09T15:56:36Z\n Ended: 2026-10-09T16:18:35Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5582720-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108567720399212544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"open","pr_sha":"9cb1004274b727280d33f5a8712c40a5ce992541","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T14:38:14Z","started_at":"2026-10-09T14:38:14Z","completed_at":"2026-10-09T15:31:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T15:41:53.233309044Z","timing_metadata_checked_at":"2026-10-09T16:03:23.863249623Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T15:41:53.236683333Z","region_metadata_checked_at":"2026-10-09T15:41:53.236683333Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fb1dc36dcbab32664cca092f4e9a6088f0bd0576c662147499c884666398e8f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108567720399212544","occurred_at":"2026-10-09T14:38:14Z","failed_at":"provision","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-09T15:19:24.628Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-09T15:19:25.146Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-09T15:19:25.146Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-09T15:19:25.687Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-10-09T15:19:25.942Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-10-09T15:19:25.942Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-09T15:19:26.093Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-09T15:25:12.869Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-09T15:25:12.869Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108564560276361216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7248,"pr_state":"open","pr_sha":"3e2810e6bdbadb284365e3e34afb765c0f03e1db","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T14:25:37Z","started_at":"2026-10-09T14:25:37Z","completed_at":"2026-10-09T16:55:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T17:04:34.785950075Z","timing_metadata_checked_at":"2026-10-09T17:04:34.785950075Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T17:04:34.789449164Z","region_metadata_checked_at":"2026-10-09T17:04:34.789449164Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"eba05bed4210ae67b858f7be46096e231152214695b6ebc972a209455641fa57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108564560276361216","occurred_at":"2026-10-09T14:25:37Z","failed_at":"alert","signature_id":"ef06df0b30d9f19d0a26d641f64c4b995322c1aeb343d1676cc66991c1e87183","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 94 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-x96nb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-x96nb has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-jtlkd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-jtlkd has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-9zkgt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-9zkgt has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-sp5cx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-sp5cx has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-cls45\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-cls45 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-v5hjd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-v5hjd has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-s8kmb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-s8kmb has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-7wbzr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-7wbzr has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-g6r2m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-g6r2m has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-rq6n8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-rq6n8 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-m29g5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-m29g5 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-pcttg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-pcttg has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-kfj5n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-kfj5n has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-8mwsk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-8mwsk has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-rdzwz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-rdzwz has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-9nrsr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-9nrsr has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-bbf4f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-bbf4f has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-xz7fq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-xz7fq has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-khnm2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-khnm2 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-tn9tb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-tn9tb has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-cks5z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-cks5z has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-lgjxz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-lgjxz has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-ghjjp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-ghjjp has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-lfpsq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-lfpsq has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-j9pqn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-j9pqn has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:05:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-5tnr6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-5tnr6 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-zhh5s\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-zhh5s has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-df2m9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-df2m9 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-d8cbr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-d8cbr has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-q4bf8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-q4bf8 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-8vkk6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-8vkk6 has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-ltf9m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-ltf9m has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-pqcmf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-pqcmf has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-qddb4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-qddb4 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-rxqr4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-rxqr4 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-8x4dn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-8x4dn has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-r4p99\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-r4p99 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-62kzb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-62kzb has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-rmkv4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-rmkv4 has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-r2blk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-r2blk has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:05:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-mzd6d\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-mzd6d has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-6qk79\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-6qk79 has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-ldl4b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-ldl4b has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-skcks\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-skcks has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-2wnf8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-2wnf8 has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-g9jdz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-g9jdz has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-zqv6n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-zqv6n has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-57999\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-57999 has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-qvtrf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-qvtrf has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-dvwv9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-dvwv9 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-dncfj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-dncfj has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:05:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-9q52l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-9q52l has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-5qtxq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-5qtxq has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:05:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-sfjfx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-sfjfx has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-4vqv7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-4vqv7 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-pvmct\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-pvmct has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-sgbt2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-sgbt2 has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-mtw9g\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-mtw9g has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-nzfck\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-nzfck has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-6n2zh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-6n2zh has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-gl292\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-gl292 has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-ptg2s\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-ptg2s has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-qvtv4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-qvtv4 has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-njvdk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-njvdk has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-5cqkh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-5cqkh has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-t8tsp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-t8tsp has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-9mtq6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-9mtq6 has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:05:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-qr84n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-qr84n has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-nxw6l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-nxw6l has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-rstls\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-rstls has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-vgqmv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-vgqmv has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-2c8gz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-2c8gz has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-9q7pr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-9q7pr has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-g7zj8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-g7zj8 has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-c9rv5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-c9rv5 has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-p6wbm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-p6wbm has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-hrfnw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-hrfnw has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:05:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-47xk7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-47xk7 has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-mspcg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-mspcg has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-qn65l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-qn65l has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-v4hhz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-v4hhz has been in a non-ready state for longer than 5 minutes.\nFiring 82:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-q4dtm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-q4dtm has been in a non-ready state for longer than 5 minutes.\nFiring 83:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-cs2fn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-cs2fn has been in a non-ready state for longer than 5 minutes.\nFiring 84:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-tv9qb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-tv9qb has been in a non-ready state for longer than 5 minutes.\nFiring 85:\n State: Resolved\n Started: 2026-10-09T15:42:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-mtp6n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-mtp6n has been in a non-ready state for longer than 5 minutes.\nFiring 86:\n State: Resolved\n Started: 2026-10-09T15:42:37Z\n Ended: 2026-10-09T16:09:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-s249v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-s249v has been in a non-ready state for longer than 5 minutes.\nFiring 87:\n State: Resolved\n Started: 2026-10-09T15:43:35Z\n Ended: 2026-10-09T16:05:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-wmvms\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-wmvms has been in a non-ready state for longer than 5 minutes.\nFiring 88:\n State: Resolved\n Started: 2026-10-09T15:44:36Z\n Ended: 2026-10-09T16:02:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-vrj6w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-vrj6w has been in a non-ready state for longer than 5 minutes.\nFiring 89:\n State: Resolved\n Started: 2026-10-09T15:44:36Z\n Ended: 2026-10-09T16:05:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-zggw8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-zggw8 has been in a non-ready state for longer than 5 minutes.\nFiring 90:\n State: Resolved\n Started: 2026-10-09T15:44:36Z\n Ended: 2026-10-09T16:05:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-5rf8m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-5rf8m has been in a non-ready state for longer than 5 minutes.\nFiring 91:\n State: Resolved\n Started: 2026-10-09T15:44:37Z\n Ended: 2026-10-09T16:05:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-8p7wx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-8p7wx has been in a non-ready state for longer than 5 minutes.\nFiring 92:\n State: Resolved\n Started: 2026-10-09T15:44:37Z\n Ended: 2026-10-09T16:05:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"network-node-identity-8449556c45-fvbcc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/network-node-identity-8449556c45-fvbcc has been in a non-ready state for longer than 5 minutes.\nFiring 93:\n State: Resolved\n Started: 2026-10-09T15:44:37Z\n Ended: 2026-10-09T16:02:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull\", pod=\"csi-snapshot-controller-8464dbc457-jmhw2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbf0ua1b1efhug6lvler0bmqlsj7ee-acr-pull/csi-snapshot-controller-8464dbc457-jmhw2 has been in a non-ready state for longer than 5 minutes.\nFiring 94:\n State: Resolved\n Started: 2026-10-09T15:44:37Z\n Ended: 2026-10-09T16:05:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6361216-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b\", pod=\"kube-controller-manager-7d8f4c9476-7t9rz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tbbemu3kdg6s9hj0na4fbg472co0emt-n5e3l6d0m8j9y7b/kube-controller-manager-7d8f4c9476-7t9rz has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"closed","pr_sha":"2be9f130d5be3c5ee0a7d837272b65a2bfadc0a0","final_merged_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T14:02:38Z","started_at":"2026-10-09T14:02:38Z","completed_at":"2026-10-09T15:27:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T15:31:15.673908827Z","timing_metadata_checked_at":"2026-10-09T15:41:53.449736459Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T15:31:15.676589319Z","region_metadata_checked_at":"2026-10-09T15:31:15.676589319Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"0eefbc314daae7be0cb8dc4ff896fc2b3b28d81106d64bf7f09a4fa7df2a3a60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","occurred_at":"2026-10-09T14:02:38Z","failed_at":"alert","signature_id":"c28dae8ce3537c2027112a365ecbd5dce6df30cf8f95741d318a215b7db8828b","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T15:20:44Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4897024/providers/microsoft.monitor/accounts/services-j4897024\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mfswx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T15:20:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4897024/providers/microsoft.monitor/accounts/services-j4897024\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mfswx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T15:20:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4897024/providers/microsoft.monitor/accounts/services-j4897024\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mfswx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-09T15:20:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.99:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4897024/providers/microsoft.monitor/accounts/services-j4897024\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mfswx\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"29d8cc086941cf609a95520e8d6a7fa62f50dedae2d8a331fd3ad7107e8e161c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","occurred_at":"2026-10-09T14:02:38Z","failed_at":"alert","signature_id":"e01049157fc58f922b04b07385b9c89d671967d9a7a5d5a68d2d2c559ee1a0da","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-09T15:02:04Z\n Ended: 2026-10-09T15:12:09Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4897024-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}},{"row_id":"60d49ede79ebd289aa592f57c9b420df0969ca57820e87deeb4e6f67137e3613","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","occurred_at":"2026-10-09T14:02:38Z","failed_at":"provision","signature_id":"24872b7f19e6b96289ddd32696a00244aac5511dd32a2c2694b7aef4a036c3f7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-09T15:06:37.853Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-09T15:06:38.448Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 values=\"map[captureMode:all clusterName:ci01-j4897024-mgmt-2 enabled:true environment:ci01 image:map[digest:sha256:11b789c31d680e7f988ca6ecb8901408801a582f560ea82572d969f5ddb243a6 registry:arohcpsvcdev.azurecr.io repository:ci-op-crcb6znc/pipeline] region:westus3 resources:map[limits:map[memory:256Mi] requests:map[cpu:20m memory:64Mi]] serviceAccount:map[name:swift-recorder]]\"\ntime=2026-10-09T15:06:38.449Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-09T15:06:38.710Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 namespace=swift-recorder\ntime=2026-10-09T15:06:39.018Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-09T15:06:39.018Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-09T15:06:39.140Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-09T15:16:42.762Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\"\ntime=2026-10-09T15:16:42.762Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d78f890a8b842ac794308d3c81fec10ddb32f68783d012d98fa9ac7172f3fde6","failure_pattern":"failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded"}},{"row_id":"b0722a811b79dc769afe92800b66a914e924c7e84a16d1430b000f1f2537181e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558761344897024","occurred_at":"2026-10-09T14:02:38Z","failed_at":"alert","signature_id":"74767212334f915bb5336f7d2acfaee3ea2d2733754d6e36e4f882ac6ca14cd5","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T15:11:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-23980774-vmss000000-slpqz\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-23980774-vmss000000-slpqz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T15:11:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-lzv6r\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-lzv6r has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T15:11:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-qbqjf\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-qbqjf has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-09T15:12:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4897024-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"swift-recorder\", pod=\"swift-recorder-6f6gx\", region=\"westus3\", severity=\"warning\"\n Description: Pod swift-recorder/swift-recorder-6f6gx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","d78f890a8b842ac794308d3c81fec10ddb32f68783d012d98fa9ac7172f3fde6","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558110011428864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"open","pr_sha":"6ea9b78a70e4c8828ec961355332d964b7902a9c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T14:00:03Z","started_at":"2026-10-09T14:00:03Z","completed_at":"2026-10-09T14:23:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T14:37:16.983849862Z","timing_metadata_checked_at":"2026-10-09T14:37:16.983849862Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-09T14:37:16.987672449Z","region_metadata_checked_at":"2026-10-09T15:41:53.331567533Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"19051b9cb6770cbbdd9ec51fa9ef59d93f4cb40a0f55dc36e340a4908c91ab9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108558110011428864","occurred_at":"2026-10-09T14:00:03Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108552522435137536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"a7588ae59ddd79bdf62a22115b0a4decd0527f7f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T13:37:50Z","started_at":"2026-10-09T13:37:50Z","completed_at":"2026-10-09T14:41:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T14:58:38.234025949Z","timing_metadata_checked_at":"2026-10-09T14:58:38.234025949Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T14:58:38.238216335Z","region_metadata_checked_at":"2026-10-09T14:58:38.238216335Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"08bafce2c1a70c1a74d14b4f859be2024cdd235f60a438c0ce62f1400364b5cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108552522435137536","occurred_at":"2026-10-09T13:37:50Z","failed_at":"provision","signature_id":"fa42174f7ac0fcceb8b39aba75fd064da702cc202084145000ae5032ccdb2c75","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-09T14:05:11.912Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-09T14:35:11.381Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-09T14:05:11.922891129Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-09T14:05:14.313036396Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-09T14:05:14.653688404Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"647a5882391a49335381fff59c5e8cedbeb838f7f6bedacd6500d31f262d8c2e","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j5137536-mgmt-2\",\"clusterName\":\"ci01-j513753","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-09T14:05:11.922891129Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-09T14:05:14.313036396Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-09T14:05:14.653688404Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5137536-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5137536-mgmt-2\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}},{"row_id":"ef354a875060555ed8a43cb50552750f184c8d9312bc22048f1d44ccc17cec38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108552522435137536","occurred_at":"2026-10-09T13:37:50Z","failed_at":"alert","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T14:26:58Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j5137536-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["647a5882391a49335381fff59c5e8cedbeb838f7f6bedacd6500d31f262d8c2e","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108538835301306368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7401,"pr_state":"open","pr_sha":"4a7dc850f3e0aedce029845c0b08fb31ecdd3f3e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T12:43:27Z","started_at":"2026-10-09T12:43:27Z","completed_at":"2026-10-09T15:09:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T15:20:32.032740268Z","timing_metadata_checked_at":"2026-10-09T15:20:32.032740268Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T15:20:32.041160042Z","region_metadata_checked_at":"2026-10-09T15:20:32.041160042Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7364d83a13a4559548897d28e4264c168ff0eb36a68dde8314697b7336c10cfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108538835301306368","occurred_at":"2026-10-09T12:43:27Z","failed_at":"e2e","signature_id":"3391ca8dbe2cfc6a644473fafbfda82e44265275df5a1354a12ae66728b85887","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xfc204541c00\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-j844k7kwk994\" to finish creating: GET https://rp.j1306368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47be8315-7c7b-4770-bede-fbd3e338f83d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47be8315-7c7b-4770-bede-fbd3e338f83d\",\n \"name\": \"47be8315-7c7b-4770-bede-fbd3e338f83d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T14:07:16.315050699Z\",\n \"endTime\": \"2026-10-09T14:36:17.115068925Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c8a6f8l6z1i8v1z-cordon-np-szltg-ptx4m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-j844k7kwk994--managed/providers/Microsoft.Compute/virtualMachines/c8a6f8l6z1i8v1z-cordon-np-szltg-ptx4m to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c8a6f8l6z1i8v1z-cordon-np-szltg-ptx4m: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-j844k7kwk994--managed/providers/Microsoft.Compute/virtualMachines/c8a6f8l6z1i8v1z-cordon-np-szltg-ptx4m to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xfc204541c00\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-j844k7kwk994\" to finish creating: GET https://rp.j1306368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47be8315-7c7b-4770-bede-fbd3e338f83d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47be8315-7c7b-4770-bede-fbd3e338f83d\",\n \"name\": \"47be8315-7c7b-4770-bede-fbd3e338f83d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T14:07:16.315050699Z\",\n \"endTime\": \"2026-10-09T14:36:17.115068925Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c8a6f8l6z1i8v1z-cordon-np-szltg-ptx4m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-j844k7kwk994--managed/providers/Microsoft.Compute/virtualMachines/c8a6f8l6z1i8v1z-cordon-np-szltg-ptx4m to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c8a6f8l6z1i8v1z-cordon-np-szltg-ptx4m: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-j844k7kwk994--managed/providers/Microsoft.Compute/virtualMachines/c8a6f8l6z1i8v1z-cordon-np-szltg-ptx4m to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108529479205785600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"closed","pr_sha":"4ada4bfca590fd84b0b6be17befd2127f1df09bf","final_merged_sha":"96338d0d4c9fe0829812f6efeebdd1fea8adc217","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T12:05:17Z","started_at":"2026-10-09T12:05:17Z","completed_at":"2026-10-09T14:56:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T15:09:36.405238407Z","timing_metadata_checked_at":"2026-10-09T15:09:36.405238407Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T15:09:36.408496897Z","region_metadata_checked_at":"2026-10-09T15:09:36.408496897Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108526653842919424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"open","pr_sha":"19fe5aa6cc25cb385c23d1fbcd19f013bba773cf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T11:55:03Z","started_at":"2026-10-09T11:55:03Z","completed_at":"2026-10-09T14:12:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T14:26:22.489267842Z","timing_metadata_checked_at":"2026-10-09T14:26:22.489267842Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T14:26:22.493778427Z","region_metadata_checked_at":"2026-10-09T14:26:22.493778427Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7437,"pr_state":"closed","pr_sha":"514bd1a701cbf1807b2ef8dbef4df93cdc052987","final_merged_sha":"7495c811b9a906fee2419ebd6d9df2930140dd35","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T11:02:01Z","started_at":"2026-10-09T11:02:01Z","completed_at":"2026-10-09T13:24:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T13:35:16.197508278Z","timing_metadata_checked_at":"2026-10-09T13:35:16.197508278Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T13:35:16.200706968Z","region_metadata_checked_at":"2026-10-09T13:35:16.200706968Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0320aae6b9b5c52a99d34b93abb206da50a479d5f98f21f18a326f08cc218876","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","occurred_at":"2026-10-09T11:02:01Z","failed_at":"alert","signature_id":"95e8facc46b0421aa919d496d5b63d8a35fd8772adf859c61c4dbd4480045aea","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T12:59:03Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2706688-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.243:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2706688/providers/microsoft.monitor/accounts/hcps-j2706688\", namespace=\"ocm-arohcpci01-2tb8dp1rrd7cqhltc59rk3l5npdng5m4-f9l2t6z5w8u0h6h\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xtmdj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tb8dp1rrd7cqhltc59rk3l5npdng5m4-f9l2t6z5w8u0h6h/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T13:00:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2706688-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.243:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2706688/providers/microsoft.monitor/accounts/hcps-j2706688\", namespace=\"ocm-arohcpci01-2tb8dp1rrd7cqhltc59rk3l5npdng5m4-f9l2t6z5w8u0h6h\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xtmdj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tb8dp1rrd7cqhltc59rk3l5npdng5m4-f9l2t6z5w8u0h6h/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"26f4ac3b2e73457d6ce190833049fb520aae0190071052c619a285140b615006","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","occurred_at":"2026-10-09T11:02:01Z","failed_at":"e2e","signature_id":"cbfda8e35f43a33bada09d676dd317e056f0425610ce818b8ca139686e99b4bb","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x87a272e80c0\u003e: \n resource group \"autoscaling-cluster-dch49t4bvhmd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dch49t4bvhmd\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/314e6611-87cc-467f-b3e8-5c7bded86eeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/314e6611-87cc-467f-b3e8-5c7bded86eeb\",\n \"name\": \"314e6611-87cc-467f-b3e8-5c7bded86eeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:11:02.340187908Z\",\n \"endTime\": \"2026-10-09T12:35:03.752595501Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x87a272e80c0\u003e: \n resource group \"autoscaling-cluster-dch49t4bvhmd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-dch49t4bvhmd\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/314e6611-87cc-467f-b3e8-5c7bded86eeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/314e6611-87cc-467f-b3e8-5c7bded86eeb\",\n \"name\": \"314e6611-87cc-467f-b3e8-5c7bded86eeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:11:02.340187908Z\",\n \"endTime\": \"2026-10-09T12:35:03.752595501Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ef0ad2faa86ab70e9b7ec605c0698308ac262c895472528db1f8276133c25903","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","occurred_at":"2026-10-09T11:02:01Z","failed_at":"e2e","signature_id":"33c8fe82065035c70356f5c9aae13601f066c147beb33cc8c02dde9d53b03f2f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2403780381b0\u003e: \n resource group \"rg-candidate-4-22-2twmlnh2pntd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-97w8j7\" in resourcegroup=\"rg-candidate-4-22-2twmlnh2pntd\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec134b00-506f-4598-8f43-9039b6949cbf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec134b00-506f-4598-8f43-9039b6949cbf\",\n \"name\": \"ec134b00-506f-4598-8f43-9039b6949cbf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:19:56.634862294Z\",\n \"endTime\": \"2026-10-09T12:44:04.410412622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2403780381b0\u003e: \n resource group \"rg-candidate-4-22-2twmlnh2pntd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-97w8j7\" in resourcegroup=\"rg-candidate-4-22-2twmlnh2pntd\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec134b00-506f-4598-8f43-9039b6949cbf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec134b00-506f-4598-8f43-9039b6949cbf\",\n \"name\": \"ec134b00-506f-4598-8f43-9039b6949cbf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:19:56.634862294Z\",\n \"endTime\": \"2026-10-09T12:44:04.410412622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3409f27655c895f2707509c93a6cec4de2066378613e1e017f3543a3ae96f19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513309102706688","occurred_at":"2026-10-09T11:02:01Z","failed_at":"e2e","signature_id":"fd5af6bc3f19a3de8c7e9853e809e76590e6bfa6137c67e04556267aa2c73d5b","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3f620761c090\u003e: \n resource group \"customer-rg-lbqh9vzm8skr\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lbqh9vzm8skr\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff9dce59-52f3-4407-ad07-f8872d204253\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff9dce59-52f3-4407-ad07-f8872d204253\",\n \"name\": \"ff9dce59-52f3-4407-ad07-f8872d204253\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:06:09.323431375Z\",\n \"endTime\": \"2026-10-09T12:30:13.733939626Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3f620761c090\u003e: \n resource group \"customer-rg-lbqh9vzm8skr\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-lbqh9vzm8skr\" to finish deleting: GET https://rp.j2706688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff9dce59-52f3-4407-ad07-f8872d204253\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ff9dce59-52f3-4407-ad07-f8872d204253\",\n \"name\": \"ff9dce59-52f3-4407-ad07-f8872d204253\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:06:09.323431375Z\",\n \"endTime\": \"2026-10-09T12:30:13.733939626Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"closed","pr_sha":"2be9f130d5be3c5ee0a7d837272b65a2bfadc0a0","final_merged_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T11:01:56Z","started_at":"2026-10-09T11:01:56Z","completed_at":"2026-10-09T13:25:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T13:35:16.020746838Z","timing_metadata_checked_at":"2026-10-09T13:35:16.020746838Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T13:35:16.023907928Z","region_metadata_checked_at":"2026-10-09T13:35:16.023907928Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"4c45c16d3a59a9826cb52842864dc2abb83fa9796e99dea1222638f8fa21c706","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","failed_at":"e2e","signature_id":"4806f8d967c169f14897e23c9636721ed45b628d0224580a25d7df612eeaf06d","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x14de9654a318\u003e: \n resource group \"arm64-vm-cluster-bbmplmj7xnjh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bbmplmj7xnjh\" to finish deleting: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\",\n \"name\": \"8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:10:18.071494213Z\",\n \"endTime\": \"2026-10-09T12:34:22.504221789Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x14de9654a318\u003e: \n resource group \"arm64-vm-cluster-bbmplmj7xnjh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bbmplmj7xnjh\" to finish deleting: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\",\n \"name\": \"8789fff8-eb02-48e7-adfa-7a5bedc2a9ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:10:18.071494213Z\",\n \"endTime\": \"2026-10-09T12:34:22.504221789Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5b5d71d3572610efb614edda02f48c6b9e526c94e0545635c9eb54623f877131","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","failed_at":"e2e","signature_id":"b153d83c712b7a426cea9f27f54a827e240c35a8f024ef0bde94a81ade36c37b","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x58c914fe040\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-4zl2jn5xrcmj\" to finish creating: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4fdf8129-ec4e-4f54-a67c-3179e8542d77\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4fdf8129-ec4e-4f54-a67c-3179e8542d77\",\n \"name\": \"4fdf8129-ec4e-4f54-a67c-3179e8542d77\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:01:50.770272194Z\",\n \"endTime\": \"2026-10-09T12:30:51.906223646Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-qf6rt-prsfz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-4zl2jn5xrcmj--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qf6rt-prsfz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-qf6rt-prsfz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-4zl2jn5xrcmj--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qf6rt-prsfz to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x58c914fe040\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-4zl2jn5xrcmj\" to finish creating: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4fdf8129-ec4e-4f54-a67c-3179e8542d77\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4fdf8129-ec4e-4f54-a67c-3179e8542d77\",\n \"name\": \"4fdf8129-ec4e-4f54-a67c-3179e8542d77\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:01:50.770272194Z\",\n \"endTime\": \"2026-10-09T12:30:51.906223646Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-qf6rt-prsfz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-4zl2jn5xrcmj--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qf6rt-prsfz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-qf6rt-prsfz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-4zl2jn5xrcmj--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-qf6rt-prsfz to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"671d7114b7a463b1ce6ce5ad8c5c190bb52887817783a152204a0cecf9b7e34a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","failed_at":"e2e","signature_id":"3c56fcfecfdfd7699d62c207a6ece2f62b01d37359d14f8eaa7e27e10724c8d1","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ae2e5f963c0\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2c747kttkrf7\": PUT https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2c747kttkrf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ae2e5f963c0\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2c747kttkrf7\": PUT https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2c747kttkrf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"67a946db7cb5344d284daabfe804a132022f313e3f1fafeeee6625a8af6061b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","failed_at":"e2e","signature_id":"bbfbd4cef85a808bf51623d1118bb2a4b11c9f5723a7f47496489be8ca80a6ce","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xdad5eeb8138\u003e: \n resource group \"external-auth-cluster-cmj7tlkjf2f4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-cmj7tlkjf2f4\" to finish deleting: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7900d0f-9832-43df-824c-157a3931804e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7900d0f-9832-43df-824c-157a3931804e\",\n \"name\": \"a7900d0f-9832-43df-824c-157a3931804e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:11:18.997805773Z\",\n \"endTime\": \"2026-10-09T12:35:21.856574986Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xdad5eeb8138\u003e: \n resource group \"external-auth-cluster-cmj7tlkjf2f4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-cmj7tlkjf2f4\" to finish deleting: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7900d0f-9832-43df-824c-157a3931804e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a7900d0f-9832-43df-824c-157a3931804e\",\n \"name\": \"a7900d0f-9832-43df-824c-157a3931804e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:11:18.997805773Z\",\n \"endTime\": \"2026-10-09T12:35:21.856574986Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b981ca5992f5969d88e864b6414d14fdefdb936c2309ced17fceb9b3a034f72e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108513285430054912","occurred_at":"2026-10-09T11:01:56Z","failed_at":"e2e","signature_id":"b4cf9fb75f372d37e5b213fd1d972d3a7a4a75135dc7f43f2e40572b1ca8a71b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x354d3c777620\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-78sxml\" in resourcegroup=\"rg-np-skip-minor-78sxml-jbc8sq5wwff6\" to finish creating: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4cdfa78a-a3ff-4b30-a50e-b80eb52d2843\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4cdfa78a-a3ff-4b30-a50e-b80eb52d2843\",\n \"name\": \"4cdfa78a-a3ff-4b30-a50e-b80eb52d2843\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:30:48.28195188Z\",\n \"endTime\": \"2026-10-09T12:59:50.51236808Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d0c9p8p4u4g1h6f-nps-420-422-9z6pl-fdr4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-78sxml-jbc8sq5wwff6-np-sm-78sxml--managed/providers/Microsoft.Compute/virtualMachines/d0c9p8p4u4g1h6f-nps-420-422-9z6pl-fdr4n to exist\\nMachine d0c9p8p4u4g1h6f-nps-420-422-9z6pl-kmv4b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-78sxml-jbc8sq5wwff6-np-sm-78sxml--managed/providers/Microsoft.Compute/virtualMachines/d0c9p8p4u4g1h6f-nps-420-422-9z6pl-kmv4b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d0c9p8p4u4g1h6f-nps-420-422-9z6pl-fdr4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-78sxml-jbc8sq5wwff6-np-sm-78sxml--managed/providers/Microsoft.Compute/virtualMachines/d0c9p8p4u4g1h6f-nps-420-422-9z6pl-fdr4n to exist\\nMachine d0c9p8p4u4g1h6f-nps-420-422-9z6pl-kmv4b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-78sxml-jbc8sq5wwff6-np-sm-78sxml--managed/providers/Microsoft.Compute/virtualMachines/d0c9p8p4u4g1h6f-nps-420-422-9z6pl-kmv4b to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x354d3c777620\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-78sxml\" in resourcegroup=\"rg-np-skip-minor-78sxml-jbc8sq5wwff6\" to finish creating: GET https://rp.j0054912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4cdfa78a-a3ff-4b30-a50e-b80eb52d2843\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4cdfa78a-a3ff-4b30-a50e-b80eb52d2843\",\n \"name\": \"4cdfa78a-a3ff-4b30-a50e-b80eb52d2843\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:30:48.28195188Z\",\n \"endTime\": \"2026-10-09T12:59:50.51236808Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d0c9p8p4u4g1h6f-nps-420-422-9z6pl-fdr4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-78sxml-jbc8sq5wwff6-np-sm-78sxml--managed/providers/Microsoft.Compute/virtualMachines/d0c9p8p4u4g1h6f-nps-420-422-9z6pl-fdr4n to exist\\nMachine d0c9p8p4u4g1h6f-nps-420-422-9z6pl-kmv4b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-78sxml-jbc8sq5wwff6-np-sm-78sxml--managed/providers/Microsoft.Compute/virtualMachines/d0c9p8p4u4g1h6f-nps-420-422-9z6pl-kmv4b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d0c9p8p4u4g1h6f-nps-420-422-9z6pl-fdr4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-78sxml-jbc8sq5wwff6-np-sm-78sxml--managed/providers/Microsoft.Compute/virtualMachines/d0c9p8p4u4g1h6f-nps-420-422-9z6pl-fdr4n to exist\\nMachine d0c9p8p4u4g1h6f-nps-420-422-9z6pl-kmv4b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-78sxml-jbc8sq5wwff6-np-sm-78sxml--managed/providers/Microsoft.Compute/virtualMachines/d0c9p8p4u4g1h6f-nps-420-422-9z6pl-kmv4b to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7441,"pr_state":"open","pr_sha":"f4a4d0703ec01bad16c88e642c1c33f6aa3bf758","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T11:00:31Z","started_at":"2026-10-09T11:00:31Z","completed_at":"2026-10-09T13:37:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T13:55:49.605855768Z","timing_metadata_checked_at":"2026-10-09T13:55:49.605855768Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T13:55:49.610146553Z","region_metadata_checked_at":"2026-10-09T13:55:49.610146553Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"2b259e3f5216298bc3d55070d08be2536f78fde470ac1c0454ca6faea3741a0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","failed_at":"e2e","signature_id":"02a2305632d97b7f1a474a433f8142ff13828b0ad53edb43a5c977da2a5c4bb6","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1bf97134030\u003e: \n resource group \"arm64-vm-cluster-rvxwb88jbj7x\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rvxwb88jbj7x\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2a654fd8-0468-45a6-8d9a-ee003fb30146\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2a654fd8-0468-45a6-8d9a-ee003fb30146\",\n \"name\": \"2a654fd8-0468-45a6-8d9a-ee003fb30146\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:58:55.836253279Z\",\n \"endTime\": \"2026-10-09T12:22:56.194703206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1bf97134030\u003e: \n resource group \"arm64-vm-cluster-rvxwb88jbj7x\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rvxwb88jbj7x\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2a654fd8-0468-45a6-8d9a-ee003fb30146\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2a654fd8-0468-45a6-8d9a-ee003fb30146\",\n \"name\": \"2a654fd8-0468-45a6-8d9a-ee003fb30146\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:58:55.836253279Z\",\n \"endTime\": \"2026-10-09T12:22:56.194703206Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"350b06fe6d2e4700dcb821f9de0e9543338ab8b6310c3cb1744ff3a5f9f12c0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","failed_at":"e2e","signature_id":"6c94a10ce89058dfd74a3fdd386049b899cca67c20e5c87b6187a4e4e622cee2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1d94d9c80a8\u003e: \n resource group \"autoscaling-cluster-tj5rdtq5zbg6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tj5rdtq5zbg6\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/962a68bc-b321-487b-b276-789b2307a373\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/962a68bc-b321-487b-b276-789b2307a373\",\n \"name\": \"962a68bc-b321-487b-b276-789b2307a373\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:58:48.91579633Z\",\n \"endTime\": \"2026-10-09T12:22:56.421671364Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1d94d9c80a8\u003e: \n resource group \"autoscaling-cluster-tj5rdtq5zbg6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-tj5rdtq5zbg6\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/962a68bc-b321-487b-b276-789b2307a373\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/962a68bc-b321-487b-b276-789b2307a373\",\n \"name\": \"962a68bc-b321-487b-b276-789b2307a373\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:58:48.91579633Z\",\n \"endTime\": \"2026-10-09T12:22:56.421671364Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5d81163909a6972dc4ae0457c1648dce0adea2e8b3ad992435c3df06f7175865","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","failed_at":"e2e","signature_id":"558c8187820f4c407e61afc39a35aa683527faaae3460d474421fd2b801bb83c","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x11708fbe8558\u003e: \n resource group \"patch-tags-qgj9df44rmxf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-qgj9df44rmxf\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dae7a46b-a56b-43d2-8533-c8bfc2bffae7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dae7a46b-a56b-43d2-8533-c8bfc2bffae7\",\n \"name\": \"dae7a46b-a56b-43d2-8533-c8bfc2bffae7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:53:27.886110615Z\",\n \"endTime\": \"2026-10-09T12:17:38.071711819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb8a5khcdtm17dl9aq7kql2ocecgpqo still exists (deletion dispatched at 2026-10-09T11:53:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x11708fbe8558\u003e: \n resource group \"patch-tags-qgj9df44rmxf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-qgj9df44rmxf\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dae7a46b-a56b-43d2-8533-c8bfc2bffae7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dae7a46b-a56b-43d2-8533-c8bfc2bffae7\",\n \"name\": \"dae7a46b-a56b-43d2-8533-c8bfc2bffae7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:53:27.886110615Z\",\n \"endTime\": \"2026-10-09T12:17:38.071711819Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb8a5khcdtm17dl9aq7kql2ocecgpqo still exists (deletion dispatched at 2026-10-09T11:53:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8cac2869bca06abb81005760b84ac76ce94ce3b50a9f0b04160680717135494d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","failed_at":"e2e","signature_id":"2e5e4c7c8eb64f44eb3b1af95f10974b102ff0871033df3fb3fbe2fb1f9323bb","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x282f78936210\u003e: \n resource group \"admin-api-breakglass-pwnj2vwmx94x\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-pwnj2vwmx94x\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe1d31a-a076-408d-a7e6-045a7416df72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe1d31a-a076-408d-a7e6-045a7416df72\",\n \"name\": \"efe1d31a-a076-408d-a7e6-045a7416df72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:54:04.049690376Z\",\n \"endTime\": \"2026-10-09T12:18:06.189008845Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x282f78936210\u003e: \n resource group \"admin-api-breakglass-pwnj2vwmx94x\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-pwnj2vwmx94x\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe1d31a-a076-408d-a7e6-045a7416df72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efe1d31a-a076-408d-a7e6-045a7416df72\",\n \"name\": \"efe1d31a-a076-408d-a7e6-045a7416df72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:54:04.049690376Z\",\n \"endTime\": \"2026-10-09T12:18:06.189008845Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b23a68e2d300a1b4d517a398115eb86468046b440910751e0e8dbee30f1920e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","failed_at":"e2e","signature_id":"da09c37462afa5025b563af86b7fe1d3b933772f65cffb7c0600dea0aa30b85d","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x715418a8198\u003e: \n resource group \"rg-hs-pre--d8fntzv77rg5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-6jnw6j\" in resourcegroup=\"rg-hs-pre--d8fntzv77rg5\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f6364b4-168b-4408-93de-ce5f903b20aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f6364b4-168b-4408-93de-ce5f903b20aa\",\n \"name\": \"1f6364b4-168b-4408-93de-ce5f903b20aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:05:02.733087351Z\",\n \"endTime\": \"2026-10-09T12:29:06.620380667Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb89kg3h59n4o9tks40jdsh5l5jro8g still exists (deletion dispatched at 2026-10-09T12:05:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x715418a8198\u003e: \n resource group \"rg-hs-pre--d8fntzv77rg5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-6jnw6j\" in resourcegroup=\"rg-hs-pre--d8fntzv77rg5\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f6364b4-168b-4408-93de-ce5f903b20aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1f6364b4-168b-4408-93de-ce5f903b20aa\",\n \"name\": \"1f6364b4-168b-4408-93de-ce5f903b20aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:05:02.733087351Z\",\n \"endTime\": \"2026-10-09T12:29:06.620380667Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb89kg3h59n4o9tks40jdsh5l5jro8g still exists (deletion dispatched at 2026-10-09T12:05:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c7b446697587cc84e3a856182dc254523a69f81bbf4157f45e6e735ac9cf461d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","failed_at":"e2e","signature_id":"cc670ab2e0f9fbf285a3decb00d2ef7f9ebf1bed8c73fe577bdc097fa7b42136","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3c8ada036198\u003e: \n resource group \"rg-candidate-4-22-7xk4vqr4kmpg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-68b4x4\" in resourcegroup=\"rg-candidate-4-22-7xk4vqr4kmpg\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25b3de70-92ee-47c8-9155-8c0ca5a96889\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25b3de70-92ee-47c8-9155-8c0ca5a96889\",\n \"name\": \"25b3de70-92ee-47c8-9155-8c0ca5a96889\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:01:27.302551031Z\",\n \"endTime\": \"2026-10-09T12:25:36.409770296Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb89kods8necs5a4oar2jo8ji1411tb still exists (deletion dispatched at 2026-10-09T12:01:28Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3c8ada036198\u003e: \n resource group \"rg-candidate-4-22-7xk4vqr4kmpg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-68b4x4\" in resourcegroup=\"rg-candidate-4-22-7xk4vqr4kmpg\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25b3de70-92ee-47c8-9155-8c0ca5a96889\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/25b3de70-92ee-47c8-9155-8c0ca5a96889\",\n \"name\": \"25b3de70-92ee-47c8-9155-8c0ca5a96889\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:01:27.302551031Z\",\n \"endTime\": \"2026-10-09T12:25:36.409770296Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb89kods8necs5a4oar2jo8ji1411tb still exists (deletion dispatched at 2026-10-09T12:01:28Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e862dbd1202767e66c29a0d9978dbbcc43bd09aaaca0c842f67372050e9c351b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108512930990395392","occurred_at":"2026-10-09T11:00:31Z","failed_at":"e2e","signature_id":"bd0bca41ebd669a1d4d1dbf0424b19b876b1622f03854960d3114e98009f90b0","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x330cf0c08078\u003e: \n resource group \"kusto-logs-dxkfj4jt2n4s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-dxkfj4jt2n4s\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/735a1c9a-7d59-48f3-9927-25e5cda56c30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/735a1c9a-7d59-48f3-9927-25e5cda56c30\",\n \"name\": \"735a1c9a-7d59-48f3-9927-25e5cda56c30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:55:18.945459438Z\",\n \"endTime\": \"2026-10-09T12:19:26.457578493Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb8a54asvaa4eaak0rehljj280alpei still exists (deletion dispatched at 2026-10-09T11:55:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x330cf0c08078\u003e: \n resource group \"kusto-logs-dxkfj4jt2n4s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-dxkfj4jt2n4s\" to finish deleting: GET https://rp.j0395392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/735a1c9a-7d59-48f3-9927-25e5cda56c30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/735a1c9a-7d59-48f3-9927-25e5cda56c30\",\n \"name\": \"735a1c9a-7d59-48f3-9927-25e5cda56c30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:55:18.945459438Z\",\n \"endTime\": \"2026-10-09T12:19:26.457578493Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb8a54asvaa4eaak0rehljj280alpei still exists (deletion dispatched at 2026-10-09T11:55:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"open","pr_sha":"88f03d157b7ae5aeb1cb160d9dc0324f4f088e79","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T10:55:40Z","started_at":"2026-10-09T10:55:40Z","completed_at":"2026-10-09T13:10:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T13:24:50.696820129Z","timing_metadata_checked_at":"2026-10-09T13:24:50.696820129Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T13:24:50.700657916Z","region_metadata_checked_at":"2026-10-09T13:24:50.700657916Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"014056e688c86258019c2d7b88ed2f57d03708179372ff50665ebf6d4aa170c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"746988fc7364fd70b66c30cfad0f0919e60becbcdc601bb071a2a47a33663ad3","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x134adbf680a8\u003e: \n resource group \"rg-cp-ystream-upgrade-4-21-qjqmm9cn4czj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-z899h6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-qjqmm9cn4czj\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c0cbe6b8-69c0-4d31-bc01-953d33ab1b7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c0cbe6b8-69c0-4d31-bc01-953d33ab1b7b\",\n \"name\": \"c0cbe6b8-69c0-4d31-bc01-953d33ab1b7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:56:17.465788464Z\",\n \"endTime\": \"2026-10-09T12:20:25.844170504Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x134adbf680a8\u003e: \n resource group \"rg-cp-ystream-upgrade-4-21-qjqmm9cn4czj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-z899h6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-qjqmm9cn4czj\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c0cbe6b8-69c0-4d31-bc01-953d33ab1b7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c0cbe6b8-69c0-4d31-bc01-953d33ab1b7b\",\n \"name\": \"c0cbe6b8-69c0-4d31-bc01-953d33ab1b7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:56:17.465788464Z\",\n \"endTime\": \"2026-10-09T12:20:25.844170504Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0f90da7133b181bfd10ae59f5967e9ba53b7b1b08d8837948baa0ba2a3a44007","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"cbd3d2251edc52984c4b4261a1e14d0463ccc02836f82b3f5018bb0bc74c6def","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x362e5b21e0c0\u003e: \n resource group \"acr-pull-nb5gr6kq2hj2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-nb5gr6kq2hj2\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/743e94db-c632-4c3e-91d5-7586a4541504\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/743e94db-c632-4c3e-91d5-7586a4541504\",\n \"name\": \"743e94db-c632-4c3e-91d5-7586a4541504\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:59:58.69192494Z\",\n \"endTime\": \"2026-10-09T12:24:06.039566807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb875uo3smk0letllr5okhn9ig34svs still exists (deletion dispatched at 2026-10-09T11:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x362e5b21e0c0\u003e: \n resource group \"acr-pull-nb5gr6kq2hj2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"acr-pull\" in resourcegroup=\"acr-pull-nb5gr6kq2hj2\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/743e94db-c632-4c3e-91d5-7586a4541504\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/743e94db-c632-4c3e-91d5-7586a4541504\",\n \"name\": \"743e94db-c632-4c3e-91d5-7586a4541504\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:59:58.69192494Z\",\n \"endTime\": \"2026-10-09T12:24:06.039566807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb875uo3smk0letllr5okhn9ig34svs still exists (deletion dispatched at 2026-10-09T11:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1cea5e4b8a949ec4472da62cd48fb398260488cdc79230112accd594b34a8fc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"ca2f7f404a7338877b8aee12b28b784744b9901ec82342acf628569b744e7de9","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3346ee20e4b0\u003e: \n resource group \"feature-aggregation-vk8bg8lddcq2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vk8bg8lddcq2\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4724c843-31f5-4e70-a53a-a9dc39567964\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4724c843-31f5-4e70-a53a-a9dc39567964\",\n \"name\": \"4724c843-31f5-4e70-a53a-a9dc39567964\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:05:11.356025969Z\",\n \"endTime\": \"2026-10-09T12:29:15.890678807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3346ee20e4b0\u003e: \n resource group \"feature-aggregation-vk8bg8lddcq2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vk8bg8lddcq2\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4724c843-31f5-4e70-a53a-a9dc39567964\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4724c843-31f5-4e70-a53a-a9dc39567964\",\n \"name\": \"4724c843-31f5-4e70-a53a-a9dc39567964\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T12:05:11.356025969Z\",\n \"endTime\": \"2026-10-09T12:29:15.890678807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"239cde5518c5ac3113f0bbfa65482015b38e2ba9af87a7a018938d959d5f4624","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"85e24c7518789cbd7818e3dc2cced879b06d0785625f57d40429c77d0b688b8b","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x52f9b8920c0\u003e: \n resource group \"cni-cilium-lqwdfr6cjvht\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-lqwdfr6cjvht\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5567703-2f10-4c0b-8cf6-70ff35f674c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5567703-2f10-4c0b-8cf6-70ff35f674c9\",\n \"name\": \"d5567703-2f10-4c0b-8cf6-70ff35f674c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:57:36.843017136Z\",\n \"endTime\": \"2026-10-09T12:21:46.074977182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb86qasjq198a6vq4pknlkm4b3grgda still exists (deletion dispatched at 2026-10-09T11:57:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x52f9b8920c0\u003e: \n resource group \"cni-cilium-lqwdfr6cjvht\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-lqwdfr6cjvht\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5567703-2f10-4c0b-8cf6-70ff35f674c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5567703-2f10-4c0b-8cf6-70ff35f674c9\",\n \"name\": \"d5567703-2f10-4c0b-8cf6-70ff35f674c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:57:36.843017136Z\",\n \"endTime\": \"2026-10-09T12:21:46.074977182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb86qasjq198a6vq4pknlkm4b3grgda still exists (deletion dispatched at 2026-10-09T11:57:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"411480423c9414b62276ffe5bc4405a29b93acf14d7a883b17b88fa7b5b68aa6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"dd3b9534d3561884612cb8845e38cad44c65e45133a439f30559f4a27a034617","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x19f79d6c0108\u003e: \n resource group \"rg-candidate-4-22-qf28f2r9x98w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-zdqvrv\" in resourcegroup=\"rg-candidate-4-22-qf28f2r9x98w\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f83f8ddd-649a-4f81-bf26-922da136bfaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f83f8ddd-649a-4f81-bf26-922da136bfaf\",\n \"name\": \"f83f8ddd-649a-4f81-bf26-922da136bfaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:55:22.867180314Z\",\n \"endTime\": \"2026-10-09T12:19:26.072146961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb86qoc4q25fgh3sm80emlj62kju53i still exists (deletion dispatched at 2026-10-09T11:55:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x19f79d6c0108\u003e: \n resource group \"rg-candidate-4-22-qf28f2r9x98w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-zdqvrv\" in resourcegroup=\"rg-candidate-4-22-qf28f2r9x98w\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f83f8ddd-649a-4f81-bf26-922da136bfaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f83f8ddd-649a-4f81-bf26-922da136bfaf\",\n \"name\": \"f83f8ddd-649a-4f81-bf26-922da136bfaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:55:22.867180314Z\",\n \"endTime\": \"2026-10-09T12:19:26.072146961Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb86qoc4q25fgh3sm80emlj62kju53i still exists (deletion dispatched at 2026-10-09T11:55:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8b0328734d5f6caba287315dfab66ecf0265707b1a3a5768aba8b0cb90f73fbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"0772c78b685bdb878b9ff0b3ec87a5c4bc3da82c58ac34bdef13e588a7eb7ab1","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2b3aa8d40060\u003e: \n resource group \"private-keyvault-j49nnk79lk7l\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j49nnk79lk7l\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0feb53b8-520d-463a-b844-2af1832a77a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0feb53b8-520d-463a-b844-2af1832a77a0\",\n \"name\": \"0feb53b8-520d-463a-b844-2af1832a77a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:56:12.658651084Z\",\n \"endTime\": \"2026-10-09T12:20:16.061081704Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb87h9gah70vts6mbse5o06n385j3o0 still exists (deletion dispatched at 2026-10-09T11:56:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2b3aa8d40060\u003e: \n resource group \"private-keyvault-j49nnk79lk7l\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-j49nnk79lk7l\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0feb53b8-520d-463a-b844-2af1832a77a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0feb53b8-520d-463a-b844-2af1832a77a0\",\n \"name\": \"0feb53b8-520d-463a-b844-2af1832a77a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:56:12.658651084Z\",\n \"endTime\": \"2026-10-09T12:20:16.061081704Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb87h9gah70vts6mbse5o06n385j3o0 still exists (deletion dispatched at 2026-10-09T11:56:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bd977380c05eeb482fb28b0ff0095b0cf6dfaebb570cc2f0327ed71a22b48f14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"a607236403e65598e0d7a777b4d162c897ebc644542af7ce0b13c29ab958f2de","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3172298ee060\u003e: \n resource group \"customer-rg-85nx9txxtvjg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-85nx9txxtvjg\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c208ad27-e494-4933-aedb-2b311ea04677\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c208ad27-e494-4933-aedb-2b311ea04677\",\n \"name\": \"c208ad27-e494-4933-aedb-2b311ea04677\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:49:09.861950907Z\",\n \"endTime\": \"2026-10-09T12:13:16.248989406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb87377q9r4c48pr0vofden0ac6k3ii still exists (deletion dispatched at 2026-10-09T11:49:10Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3172298ee060\u003e: \n resource group \"customer-rg-85nx9txxtvjg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-85nx9txxtvjg\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c208ad27-e494-4933-aedb-2b311ea04677\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c208ad27-e494-4933-aedb-2b311ea04677\",\n \"name\": \"c208ad27-e494-4933-aedb-2b311ea04677\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:49:09.861950907Z\",\n \"endTime\": \"2026-10-09T12:13:16.248989406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb87377q9r4c48pr0vofden0ac6k3ii still exists (deletion dispatched at 2026-10-09T11:49:10Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e26103481a00ab4743f86459246d7c6b3c75a941441e344c5ed8bc2fc133a67d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"8d0e0342cbc9e23b844c7aca35ed99507754ea217dedb85ce43d4f14fe551007","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xb001b41e150\u003e: \n resource group \"admin-api-serialconsole-bootdiag-zkmb9sg9qvzk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-zkmb9sg9qvzk\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74bde478-0cee-426a-977f-638be37440d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74bde478-0cee-426a-977f-638be37440d5\",\n \"name\": \"74bde478-0cee-426a-977f-638be37440d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:52:47.875676815Z\",\n \"endTime\": \"2026-10-09T12:16:55.850520543Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xb001b41e150\u003e: \n resource group \"admin-api-serialconsole-bootdiag-zkmb9sg9qvzk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-zkmb9sg9qvzk\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74bde478-0cee-426a-977f-638be37440d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/74bde478-0cee-426a-977f-638be37440d5\",\n \"name\": \"74bde478-0cee-426a-977f-638be37440d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:52:47.875676815Z\",\n \"endTime\": \"2026-10-09T12:16:55.850520543Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"feab5f779fe57e195d78bf9b7a4541ff99cf8022f1f6ab5b5b5decc1ef5b67df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108511709156085760","occurred_at":"2026-10-09T10:55:40Z","failed_at":"e2e","signature_id":"d58e1b5ab93a98812a74d2fb9e1201c1960008bc7c934bc40a4972c4e065df59","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x10e954cac0c0\u003e: \n resource group \"fips-enabled-g648xvh6nzj9\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-g648xvh6nzj9\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6fa493d6-7fae-4369-b969-8577dd6dfc89\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6fa493d6-7fae-4369-b969-8577dd6dfc89\",\n \"name\": \"6fa493d6-7fae-4369-b969-8577dd6dfc89\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:52:00.540461838Z\",\n \"endTime\": \"2026-10-09T12:16:05.840029374Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x10e954cac0c0\u003e: \n resource group \"fips-enabled-g648xvh6nzj9\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-g648xvh6nzj9\" to finish deleting: GET https://rp.j6085760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6fa493d6-7fae-4369-b969-8577dd6dfc89\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6fa493d6-7fae-4369-b969-8577dd6dfc89\",\n \"name\": \"6fa493d6-7fae-4369-b969-8577dd6dfc89\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T11:52:00.540461838Z\",\n \"endTime\": \"2026-10-09T12:16:05.840029374Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108491565306482688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7226,"pr_state":"closed","pr_sha":"efca64cabcd74fa1b6a256df25900d3a5c5a8f6e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T09:35:37Z","started_at":"2026-10-09T09:35:37Z","completed_at":"2026-10-09T10:04:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T10:15:32.612569512Z","timing_metadata_checked_at":"2026-10-09T10:15:32.612569512Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T10:15:32.616695998Z","region_metadata_checked_at":"2026-10-09T10:15:32.616695998Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"40dc023b8e8b52f1c4dbca3acbea9211524576ac08872bd3dbb6566601aedb43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108491565306482688","occurred_at":"2026-10-09T09:35:37Z","failed_at":"alert","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6482688/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j6482688' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"c133662dcf45adcde39c36949a263b3f891f986486512102633d2d996b0fd5de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108491565306482688","occurred_at":"2026-10-09T09:35:37Z","failed_at":"alert","signature_id":"a5fd6e34535fa450e59098cc1740b0f19b4969d471d45588dd16e29233350a75","test_name":"[aro-hcp-observability] [infra] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6482688: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6482688/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j6482688' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"f01bbb67c999facdf67706686bcc189c47ac5daf8c95f0aa5092585e219c31e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108491565306482688","occurred_at":"2026-10-09T09:35:37Z","failed_at":"alert","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","test_name":"[aro-hcp-observability] [svc] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6482688/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j6482688' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108481553754492928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7219,"pr_state":"open","pr_sha":"39a44b4d323013cc82da4d85db8ec4120cc1c4db","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T08:55:50Z","started_at":"2026-10-09T08:55:50Z","completed_at":"2026-10-09T11:29:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T11:49:10.494148954Z","timing_metadata_checked_at":"2026-10-09T11:49:10.494148954Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T11:49:10.498520639Z","region_metadata_checked_at":"2026-10-09T11:49:10.498520639Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108480873924923392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"c5bf1e12136aa94c4727812ed1c4ec11c47d58ed","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T08:53:08Z","started_at":"2026-10-09T08:53:08Z","completed_at":"2026-10-09T11:23:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T11:38:27.898974427Z","timing_metadata_checked_at":"2026-10-09T11:38:27.898974427Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T11:38:27.903394712Z","region_metadata_checked_at":"2026-10-09T11:38:27.903394712Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108480044224483328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6944,"pr_state":"open","pr_sha":"865d18f8a5fe945f10460ee3b34947fdd30a926c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T08:49:50Z","started_at":"2026-10-09T08:49:50Z","completed_at":"2026-10-09T11:29:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T11:49:10.474344821Z","timing_metadata_checked_at":"2026-10-09T11:49:10.474344821Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T11:49:10.478247308Z","region_metadata_checked_at":"2026-10-09T11:49:10.478247308Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"349f7de64164ef7b7080928b12c4fb702df52cf59d0f7443d8e485e8e3fe4477","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108480044224483328","occurred_at":"2026-10-09T08:49:50Z","failed_at":"e2e","signature_id":"ae60c1c69c20f023c68844b91b0f521ca18d833de14f1dfc77a24752e434f5aa","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x18432eff1520\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-kfsdc7\" in resourcegroup=\"rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4\" to finish creating: GET https://rp.j4483328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ee089c8-2818-4f87-996e-9f68dab666fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ee089c8-2818-4f87-996e-9f68dab666fd\",\n \"name\": \"7ee089c8-2818-4f87-996e-9f68dab666fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T10:36:13.066941031Z\",\n \"endTime\": \"2026-10-09T11:05:15.745369037Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-j2lrd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4-np-upgrade-kf-f195e85e/providers/Microsoft.Compute/virtualMachines/o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-j2lrd to exist\\nMachine o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-qw9b7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4-np-upgrade-kf-f195e85e/providers/Microsoft.Compute/virtualMachines/o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-qw9b7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-j2lrd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4-np-upgrade-kf-f195e85e/providers/Microsoft.Compute/virtualMachines/o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-j2lrd to exist\\nMachine o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-qw9b7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4-np-upgrade-kf-f195e85e/providers/Microsoft.Compute/virtualMachines/o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-qw9b7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x18432eff1520\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-kfsdc7\" in resourcegroup=\"rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4\" to finish creating: GET https://rp.j4483328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ee089c8-2818-4f87-996e-9f68dab666fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ee089c8-2818-4f87-996e-9f68dab666fd\",\n \"name\": \"7ee089c8-2818-4f87-996e-9f68dab666fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T10:36:13.066941031Z\",\n \"endTime\": \"2026-10-09T11:05:15.745369037Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-j2lrd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4-np-upgrade-kf-f195e85e/providers/Microsoft.Compute/virtualMachines/o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-j2lrd to exist\\nMachine o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-qw9b7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4-np-upgrade-kf-f195e85e/providers/Microsoft.Compute/virtualMachines/o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-qw9b7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-j2lrd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4-np-upgrade-kf-f195e85e/providers/Microsoft.Compute/virtualMachines/o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-j2lrd to exist\\nMachine o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-qw9b7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kfsdc7-xsdgmhhbvgm4-np-upgrade-kf-f195e85e/providers/Microsoft.Compute/virtualMachines/o9h2o2o4l3l4e3q-npupgrade-4-20-x55mz-qw9b7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"64a82209f73e162c99cf1200b858a0c05de186eea1480d3b4d1b1f012558c344","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108480044224483328","occurred_at":"2026-10-09T08:49:50Z","failed_at":"e2e","signature_id":"4a88a8d89596c34e4f3ac28e02db74d5d14c30f8361d3164afdc9f87cc8dcc79","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x16e23cafa520\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-cwpvfwwq89gb\" to finish creating: GET https://rp.j4483328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e7b7c93c-6eac-40c9-80bd-d8fc4f7fb529\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e7b7c93c-6eac-40c9-80bd-d8fc4f7fb529\",\n \"name\": \"e7b7c93c-6eac-40c9-80bd-d8fc4f7fb529\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T10:23:47.421215993Z\",\n \"endTime\": \"2026-10-09T10:52:51.928906707Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c8m0p9k9w9k3x0r-np-1-xvxq5-sdcw7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-cwpvfwwq89gb--managed/providers/Microsoft.Compute/virtualMachines/c8m0p9k9w9k3x0r-np-1-xvxq5-sdcw7 to exist\\nMachine c8m0p9k9w9k3x0r-np-1-xvxq5-xf82w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-cwpvfwwq89gb--managed/providers/Microsoft.Compute/virtualMachines/c8m0p9k9w9k3x0r-np-1-xvxq5-xf82w to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c8m0p9k9w9k3x0r-np-1-xvxq5-sdcw7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-cwpvfwwq89gb--managed/providers/Microsoft.Compute/virtualMachines/c8m0p9k9w9k3x0r-np-1-xvxq5-sdcw7 to exist\\nMachine c8m0p9k9w9k3x0r-np-1-xvxq5-xf82w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-cwpvfwwq89gb--managed/providers/Microsoft.Compute/virtualMachines/c8m0p9k9w9k3x0r-np-1-xvxq5-xf82w to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x16e23cafa520\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-cwpvfwwq89gb\" to finish creating: GET https://rp.j4483328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e7b7c93c-6eac-40c9-80bd-d8fc4f7fb529\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e7b7c93c-6eac-40c9-80bd-d8fc4f7fb529\",\n \"name\": \"e7b7c93c-6eac-40c9-80bd-d8fc4f7fb529\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T10:23:47.421215993Z\",\n \"endTime\": \"2026-10-09T10:52:51.928906707Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c8m0p9k9w9k3x0r-np-1-xvxq5-sdcw7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-cwpvfwwq89gb--managed/providers/Microsoft.Compute/virtualMachines/c8m0p9k9w9k3x0r-np-1-xvxq5-sdcw7 to exist\\nMachine c8m0p9k9w9k3x0r-np-1-xvxq5-xf82w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-cwpvfwwq89gb--managed/providers/Microsoft.Compute/virtualMachines/c8m0p9k9w9k3x0r-np-1-xvxq5-xf82w to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c8m0p9k9w9k3x0r-np-1-xvxq5-sdcw7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-cwpvfwwq89gb--managed/providers/Microsoft.Compute/virtualMachines/c8m0p9k9w9k3x0r-np-1-xvxq5-sdcw7 to exist\\nMachine c8m0p9k9w9k3x0r-np-1-xvxq5-xf82w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-cwpvfwwq89gb--managed/providers/Microsoft.Compute/virtualMachines/c8m0p9k9w9k3x0r-np-1-xvxq5-xf82w to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e070c787ff15ed1e4277366628c79f5fa89f707f3792ba777a6ee0d09317e0df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108480044224483328","occurred_at":"2026-10-09T08:49:50Z","failed_at":"alert","signature_id":"2321790741ede5cfc4112f5540dcfb98866ee514350ef355bda4927b9c6a5c1d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-09T10:08:14Z\n Ended: 2026-10-09T10:15:16Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4483328-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4483328-svc).\nFiring 2:\n State: Resolved\n Started: 2026-10-09T10:31:13Z\n Ended: 2026-10-09T10:40:15Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4483328-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4483328-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108458935697543168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"closed","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T07:25:58Z","started_at":"2026-10-09T07:25:58Z","completed_at":"2026-10-09T09:42:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T09:44:46.859558769Z","timing_metadata_checked_at":"2026-10-09T09:44:46.859558769Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T09:44:46.883932292Z","region_metadata_checked_at":"2026-10-09T09:44:46.883932292Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"01681a2a9aa2dc5543b51ff7e10dc687933a918be88a8e4fbff400e9225ced75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108458935697543168","occurred_at":"2026-10-09T07:25:58Z","failed_at":"e2e","signature_id":"9c17d7d30e387ba5b6997b09f0c0488b684608a6ad4866054d6d83675e245a1b","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x126a68f88048\u003e: \n resource group \"nodepool-update-nodes-ppzw4fscb65q\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ppzw4fscb65q\" to finish deleting: GET https://rp.j7543168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5402a6f6-777b-405f-bf42-17f6202e880d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5402a6f6-777b-405f-bf42-17f6202e880d\",\n \"name\": \"5402a6f6-777b-405f-bf42-17f6202e880d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T09:00:29.162024664Z\",\n \"endTime\": \"2026-10-09T09:24:36.108080611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb5gs6ackm9riurrcuq3gadk7un22q7 still exists (deletion dispatched at 2026-10-09T09:00:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x126a68f88048\u003e: \n resource group \"nodepool-update-nodes-ppzw4fscb65q\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-ppzw4fscb65q\" to finish deleting: GET https://rp.j7543168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5402a6f6-777b-405f-bf42-17f6202e880d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5402a6f6-777b-405f-bf42-17f6202e880d\",\n \"name\": \"5402a6f6-777b-405f-bf42-17f6202e880d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T09:00:29.162024664Z\",\n \"endTime\": \"2026-10-09T09:24:36.108080611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb5gs6ackm9riurrcuq3gadk7un22q7 still exists (deletion dispatched at 2026-10-09T09:00:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452968792592384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"closed","pr_sha":"2be9f130d5be3c5ee0a7d837272b65a2bfadc0a0","final_merged_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T07:02:15Z","started_at":"2026-10-09T07:02:15Z","completed_at":"2026-10-09T09:30:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T09:34:10.151496747Z","timing_metadata_checked_at":"2026-10-09T09:34:10.151496747Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T09:34:10.155151635Z","region_metadata_checked_at":"2026-10-09T09:34:10.155151635Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"251b1502e4e553fd165ed6e1edcd099e3f4397e2f77896ad5562b6395c3985fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452968792592384","occurred_at":"2026-10-09T07:02:15Z","failed_at":"alert","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-09T08:36:10Z\n Ended: 2026-10-09T08:43:12Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2592384-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"fbcebeaa0f3759022a81eabf3ef499d13fd47e49cb50f889dbeef372bb8376ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452968792592384","occurred_at":"2026-10-09T07:02:15Z","failed_at":"e2e","signature_id":"ea6e53cca656512f02d19790e724606aed7efaeb47b605b3a33d11d36c819c38","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x95c6dd09a20\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zsv74hnjtfn5\": PUT https://rp.j2592384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-zsv74hnjtfn5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x95c6dd09a20\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-zsv74hnjtfn5\": PUT https://rp.j2592384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-zsv74hnjtfn5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"fec08a6e190900dbf6497b30a103f5a4d8ce506d62fcaf952283d55479d840ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452968792592384","occurred_at":"2026-10-09T07:02:15Z","failed_at":"e2e","signature_id":"e6756c0b7d7401645c66fdb6cab5ac9e0b763efab297c59c3a9dfd587ce7507a","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x128ebe5153a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-nnh2p9v2fmnj\" to finish creating: GET https://rp.j2592384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f161e6c5-fc32-45c6-9cfa-cf016a200caf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f161e6c5-fc32-45c6-9cfa-cf016a200caf\",\n \"name\": \"f161e6c5-fc32-45c6-9cfa-cf016a200caf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T08:16:08.568494735Z\",\n \"endTime\": \"2026-10-09T08:45:11.046514818Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-q6g8p-kt5pq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nnh2p9v2fmnj--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-q6g8p-kt5pq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-q6g8p-kt5pq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nnh2p9v2fmnj--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-q6g8p-kt5pq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x128ebe5153a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-nnh2p9v2fmnj\" to finish creating: GET https://rp.j2592384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f161e6c5-fc32-45c6-9cfa-cf016a200caf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f161e6c5-fc32-45c6-9cfa-cf016a200caf\",\n \"name\": \"f161e6c5-fc32-45c6-9cfa-cf016a200caf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T08:16:08.568494735Z\",\n \"endTime\": \"2026-10-09T08:45:11.046514818Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-q6g8p-kt5pq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nnh2p9v2fmnj--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-q6g8p-kt5pq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-q6g8p-kt5pq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-nnh2p9v2fmnj--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-q6g8p-kt5pq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452959158276096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"closed","pr_sha":"0d147555918839b4a556ae5981b12356ac4cf337","final_merged_sha":"0d147555918839b4a556ae5981b12356ac4cf337","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T07:02:13Z","started_at":"2026-10-09T07:02:13Z","completed_at":"2026-10-09T09:30:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T09:34:09.902448841Z","timing_metadata_checked_at":"2026-10-09T09:34:09.902448841Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T09:34:09.905725031Z","region_metadata_checked_at":"2026-10-09T09:34:09.905725031Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7437,"pr_state":"closed","pr_sha":"514bd1a701cbf1807b2ef8dbef4df93cdc052987","final_merged_sha":"7495c811b9a906fee2419ebd6d9df2930140dd35","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T07:01:18Z","started_at":"2026-10-09T07:01:18Z","completed_at":"2026-10-09T09:28:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T09:34:10.374252436Z","timing_metadata_checked_at":"2026-10-09T09:34:10.374252436Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T09:34:10.377814525Z","region_metadata_checked_at":"2026-10-09T09:34:10.377814525Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"33bbafb628972d758c43ad110202c39b85a3b8360b8e7c98da4be94b9f433523","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","occurred_at":"2026-10-09T07:01:18Z","failed_at":"e2e","signature_id":"71021d5ea1fe0872367c0e34afd743c532ccdcbdc2e34685178edf635e296b0d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-j492kp2w2kt8/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-j492kp2w2kt8/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-8jfp6hmxz2vs/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"8effe0af729c792f4580019311a686ad7165f4c7ab00ead3b6e64490be0c8bae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","occurred_at":"2026-10-09T07:01:18Z","failed_at":"e2e","signature_id":"997f39d2360594b30e790410b45db1739cf0adf16512701d1c5d310dca5ad763","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-w8jdcn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xd9c285fc740\u003e: \n DNS for route host agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xd9c285fc6e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-w8jdcn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xd9c285fc740\u003e: \n DNS for route host agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xd9c285fc6e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nbzfm.apps.aro.s6b6j4f3z0n0v5e.3wbk.j0006144.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mjcfp.apps.aro.y0q5g2g2n5j8w7n.wl6t.j4583424.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"ddcf73bdfd9c74cd025717acc9eacdba035426e8c40900c2a562b0de719885b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","occurred_at":"2026-10-09T07:01:18Z","failed_at":"e2e","signature_id":"d49e6094505b38289976eaa29d7017c02d2680fb4b1969d63ddd5a93cf4da739","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:495]: failed to create node pool\nUnexpected error:\n \u003c*fmt.wrapError | 0x19713c8ce520\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull\" in resourcegroup=\"acr-pull-dfz6ss2tmqmt\" to finish creating: GET https://rp.j0006144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c0e150d-aecf-4c68-b9dd-c7ae7dbf7400\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c0e150d-aecf-4c68-b9dd-c7ae7dbf7400\",\n \"name\": \"5c0e150d-aecf-4c68-b9dd-c7ae7dbf7400\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T08:12:53.421461459Z\",\n \"endTime\": \"2026-10-09T08:42:02.147015674Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:495]: failed to create node pool\nUnexpected error:\n \u003c*fmt.wrapError | 0x19713c8ce520\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull\" in resourcegroup=\"acr-pull-dfz6ss2tmqmt\" to finish creating: GET https://rp.j0006144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c0e150d-aecf-4c68-b9dd-c7ae7dbf7400\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5c0e150d-aecf-4c68-b9dd-c7ae7dbf7400\",\n \"name\": \"5c0e150d-aecf-4c68-b9dd-c7ae7dbf7400\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T08:12:53.421461459Z\",\n \"endTime\": \"2026-10-09T08:42:02.147015674Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f574a815976fa02a1842d322d84c899ebaf3ea965b8f429a1cb66fdf227ef2d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108452755680006144","occurred_at":"2026-10-09T07:01:18Z","failed_at":"alert","signature_id":"b1312384d9b2a53ea723368157d1cd9705e0e9f23816fc33749b5b53bb2fe8fb","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-09T08:14:32Z\n Ended: 2026-10-09T08:32:31Z\n Severity: Sev3\n Labels: alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0006144-svc\", component=\"fleet\", name=\"nodepoolcontroller\", region=\"westus3\", severity=\"warning\"\n Description: Fleet controller workqueue nodepoolcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108438042090934272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7292,"pr_state":"open","pr_sha":"e2249e5ba4dbcc59216d7341da5783394710eb68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T06:02:56Z","started_at":"2026-10-09T06:02:56Z","completed_at":"2026-10-09T08:20:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T08:31:39.152628489Z","timing_metadata_checked_at":"2026-10-09T08:31:39.152628489Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T08:31:39.158409471Z","region_metadata_checked_at":"2026-10-09T08:31:39.158409471Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"e0f96e78c2f4d5e85017638544cdaf46a2e75a983ca5deda70492f3c173c55e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108438042090934272","occurred_at":"2026-10-09T06:02:56Z","failed_at":"alert","signature_id":"13066145b3c69b6571c75ae201d660362eb9c50399b3ea171948ebe6c00d48cf","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T07:28:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0934272-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.212:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0934272/providers/microsoft.monitor/accounts/hcps-j0934272\", namespace=\"ocm-arohcpci01-2tb42n0fgkuoc1sge8efonl494ts6f8p-q5w0c9a9b5r6x3n\", pod=\"cloud-network-config-controller-5cf98f8c95-5bd8j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a135b966-4dab-47fb-8cc2-d75dfe7326f1\"\n Description: Pod ocm-arohcpci01-2tb42n0fgkuoc1sge8efonl494ts6f8p-q5w0c9a9b5r6x3n/cloud-network-config-controller-5cf98f8c95-5bd8j (controller) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"e589e349f0133442d1be33951884848e022e73765fe5bb5c9d7e54b5f87f9e8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108438042090934272","occurred_at":"2026-10-09T06:02:56Z","failed_at":"e2e","signature_id":"c750fcad772bcae880a5d3b350c926fbfa2d24755d777353cee4665f043e8937","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xba7a1123110\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xba7a1123110\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"f7b7f5cada183cb6e3a2f977a1843ab51deaf5e9ad0e6095f942d59ea00fc5dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108438042090934272","occurred_at":"2026-10-09T06:02:56Z","failed_at":"alert","signature_id":"f5d956fa0b22d0c0a85159f69504906557f12343cd530a352e6ae14cf54c0d8b","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T07:39:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0934272-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.212:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0934272/providers/microsoft.monitor/accounts/hcps-j0934272\", namespace=\"ocm-arohcpci01-2tb42n0fgkuoc1sge8efonl494ts6f8p-q5w0c9a9b5r6x3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qf8xh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tb42n0fgkuoc1sge8efonl494ts6f8p-q5w0c9a9b5r6x3n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T07:39:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0934272-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.212:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0934272/providers/microsoft.monitor/accounts/hcps-j0934272\", namespace=\"ocm-arohcpci01-2tb42n0fgkuoc1sge8efonl494ts6f8p-q5w0c9a9b5r6x3n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qf8xh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tb42n0fgkuoc1sge8efonl494ts6f8p-q5w0c9a9b5r6x3n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108437959073075200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7364,"pr_state":"closed","pr_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","final_merged_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T06:02:36Z","started_at":"2026-10-09T06:02:36Z","completed_at":"2026-10-09T08:24:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T08:31:39.359141651Z","timing_metadata_checked_at":"2026-10-09T08:31:39.359141651Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T08:31:39.362019042Z","region_metadata_checked_at":"2026-10-09T08:31:39.362019042Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108409414854643712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"closed","pr_sha":"0d147555918839b4a556ae5981b12356ac4cf337","final_merged_sha":"0d147555918839b4a556ae5981b12356ac4cf337","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T04:09:10Z","started_at":"2026-10-09T04:09:10Z","completed_at":"2026-10-09T06:25:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T06:36:27.537790223Z","timing_metadata_checked_at":"2026-10-09T06:36:27.537790223Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T06:36:27.54195171Z","region_metadata_checked_at":"2026-10-09T06:36:27.54195171Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"417932bda2271dceea60444976de0214cc0adb2ddd4ff7d89463e6e6765d74c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108409414854643712","occurred_at":"2026-10-09T04:09:10Z","failed_at":"alert","signature_id":"35d2f92aafdf8f1dcc0d45fecd2e1b20a6e9189a9a234f533477e9527641239a","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T05:55:14Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4643712-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.200:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4643712/providers/microsoft.monitor/accounts/hcps-j4643712\", namespace=\"ocm-arohcpci01-2tb2dcbc75fd0t83s3l86makdqbo52ni-j9n9r9c9m0a4s7v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4rlvg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tb2dcbc75fd0t83s3l86makdqbo52ni-j9n9r9c9m0a4s7v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T05:55:14Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4643712-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.200:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4643712/providers/microsoft.monitor/accounts/hcps-j4643712\", namespace=\"ocm-arohcpci01-2tb2dcbc75fd0t83s3l86makdqbo52ni-j9n9r9c9m0a4s7v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4rlvg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tb2dcbc75fd0t83s3l86makdqbo52ni-j9n9r9c9m0a4s7v/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T06:09:10Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4643712-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.200:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4643712/providers/microsoft.monitor/accounts/hcps-j4643712\", namespace=\"ocm-arohcpci01-2tb2dcbc75fd0t83s3l86makdqbo52ni-j9n9r9c9m0a4s7v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4rlvg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tb2dcbc75fd0t83s3l86makdqbo52ni-j9n9r9c9m0a4s7v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-09T06:09:10Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4643712-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.200:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4643712/providers/microsoft.monitor/accounts/hcps-j4643712\", namespace=\"ocm-arohcpci01-2tb2dcbc75fd0t83s3l86makdqbo52ni-j9n9r9c9m0a4s7v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-4rlvg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tb2dcbc75fd0t83s3l86makdqbo52ni-j9n9r9c9m0a4s7v/cluster-api has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"8972c63e91d0a5b8f0d63428d98b48f23704a851be4f582693f0f4142c1d0e69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108409414854643712","occurred_at":"2026-10-09T04:09:10Z","failed_at":"e2e","signature_id":"683f0dfc089fd69b29c305b21fb888467085c261c34597202abd78d953a0cf3b","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2f3c0aa90108\u003e: \n resource group \"admin-credential-lifecycle-test-lxkxdsm95v5q\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-xck96d\" in resourcegroup=\"admin-credential-lifecycle-test-lxkxdsm95v5q\" to finish deleting: GET https://rp.j4643712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d7634497-b76b-417d-ba61-d17625706ddd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d7634497-b76b-417d-ba61-d17625706ddd\",\n \"name\": \"d7634497-b76b-417d-ba61-d17625706ddd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T05:12:01.285111592Z\",\n \"endTime\": \"2026-10-09T05:36:09.092898944Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb2dcbc75fd0t83s3l86makdqbo52ni still exists (deletion dispatched at 2026-10-09T05:12:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2f3c0aa90108\u003e: \n resource group \"admin-credential-lifecycle-test-lxkxdsm95v5q\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-xck96d\" in resourcegroup=\"admin-credential-lifecycle-test-lxkxdsm95v5q\" to finish deleting: GET https://rp.j4643712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d7634497-b76b-417d-ba61-d17625706ddd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d7634497-b76b-417d-ba61-d17625706ddd\",\n \"name\": \"d7634497-b76b-417d-ba61-d17625706ddd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T05:12:01.285111592Z\",\n \"endTime\": \"2026-10-09T05:36:09.092898944Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb2dcbc75fd0t83s3l86makdqbo52ni still exists (deletion dispatched at 2026-10-09T05:12:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7438/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108402844888469504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7438,"pr_state":"closed","pr_sha":"d8ea15b466aad1fe925f183111c67bae95fac5c8","final_merged_sha":"d8ea15b466aad1fe925f183111c67bae95fac5c8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T03:43:05Z","started_at":"2026-10-09T03:43:05Z","completed_at":"2026-10-09T05:57:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T06:15:38.301411233Z","timing_metadata_checked_at":"2026-10-09T06:15:38.301411233Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T06:15:38.305740419Z","region_metadata_checked_at":"2026-10-09T06:15:38.305740419Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0fb6e2bf889646a5bb345ea85a4af0b5fd53963d6b4e2d39ec5a99599f2b7b3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7438/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108402844888469504","occurred_at":"2026-10-09T03:43:05Z","failed_at":"e2e","signature_id":"b15702fdd5528fda136f82def4e6c124ec913c615faaa8a57b58834afb1e560b","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-4-21-cxd2vj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b58d6261e60\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-4-21-cxd2vj\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-21-hw27qjjdkxw6\" to finish creating: GET https://rp.j8469504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77583a7d-a965-4558-a265-ffd7e06f41b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77583a7d-a965-4558-a265-ffd7e06f41b9\",\n \"name\": \"77583a7d-a965-4558-a265-ffd7e06f41b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T04:44:07.077624496Z\",\n \"endTime\": \"2026-10-09T05:13:07.452067539Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a9a1f6e4r7f8b6e-np-deny-pods-j2b5z-btsbp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-4-21-hw27qjjdkxw6-cp-ystream-de-81b4ef58/providers/Microsoft.Compute/virtualMachines/a9a1f6e4r7f8b6e-np-deny-pods-j2b5z-btsbp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a9a1f6e4r7f8b6e-np-deny-pods-j2b5z-btsbp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-4-21-hw27qjjdkxw6-cp-ystream-de-81b4ef58/providers/Microsoft.Compute/virtualMachines/a9a1f6e4r7f8b6e-np-deny-pods-j2b5z-btsbp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-4-21-cxd2vj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b58d6261e60\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-4-21-cxd2vj\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-21-hw27qjjdkxw6\" to finish creating: GET https://rp.j8469504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77583a7d-a965-4558-a265-ffd7e06f41b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/77583a7d-a965-4558-a265-ffd7e06f41b9\",\n \"name\": \"77583a7d-a965-4558-a265-ffd7e06f41b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T04:44:07.077624496Z\",\n \"endTime\": \"2026-10-09T05:13:07.452067539Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a9a1f6e4r7f8b6e-np-deny-pods-j2b5z-btsbp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-4-21-hw27qjjdkxw6-cp-ystream-de-81b4ef58/providers/Microsoft.Compute/virtualMachines/a9a1f6e4r7f8b6e-np-deny-pods-j2b5z-btsbp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a9a1f6e4r7f8b6e-np-deny-pods-j2b5z-btsbp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-4-21-hw27qjjdkxw6-cp-ystream-de-81b4ef58/providers/Microsoft.Compute/virtualMachines/a9a1f6e4r7f8b6e-np-deny-pods-j2b5z-btsbp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6452/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108386783350755328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6452,"pr_state":"open","pr_sha":"47134eab86ea7953ac2c086b94278a2c59202a47","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T02:39:15Z","started_at":"2026-10-09T02:39:15Z","completed_at":"2026-10-09T04:53:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T05:12:39.995472371Z","timing_metadata_checked_at":"2026-10-09T05:12:39.995472371Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T05:12:39.999237059Z","region_metadata_checked_at":"2026-10-09T05:12:39.999237059Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108385613169299456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7394,"pr_state":"open","pr_sha":"c0d006cc0bb4f9012ea66a764ec0977c60acc0b7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T02:34:36Z","started_at":"2026-10-09T02:34:36Z","completed_at":"2026-10-09T03:37:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T03:48:59.196070808Z","timing_metadata_checked_at":"2026-10-09T03:48:59.196070808Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T03:48:59.20179969Z","region_metadata_checked_at":"2026-10-09T03:48:59.20179969Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"3a8232199552b11839463da1c36d515f24de4f3ae75602320519eaf63e6ce049","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108385613169299456","occurred_at":"2026-10-09T02:34:36Z","failed_at":"provision","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-09T03:02:25.127Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-09T03:02:27.026Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-09T03:02:29.104Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9100c383496433b8981e04307d4ff94d8b8595203688da3bb283a6fce4e49fea portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9299456-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9100c383496433b8981e04307d4ff94d8b8595203688da3bb283a6fce4e49fea\ntime=2026-10-09T03:32:24.844Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}},{"row_id":"b5f29be2e0af9df3a4c6087618b227813d3f5f89b4436c3f5b6e1db99cc007c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108385613169299456","occurred_at":"2026-10-09T02:34:36Z","failed_at":"alert","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T03:29:55Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j9299456-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377968924430336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7414,"pr_state":"closed","pr_sha":"681d05374ebd4c69860a56d55814d585d94e8691","final_merged_sha":"681d05374ebd4c69860a56d55814d585d94e8691","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T02:04:14Z","started_at":"2026-10-09T02:04:14Z","completed_at":"2026-10-09T04:29:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T04:41:24.698981413Z","timing_metadata_checked_at":"2026-10-09T04:41:24.698981413Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T04:41:24.703079199Z","region_metadata_checked_at":"2026-10-09T04:41:24.703079199Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2ad5fd9838a316132aa5b69813e33e84c20fa6ad2e4f09d939c47084c46c4312","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377968924430336","occurred_at":"2026-10-09T02:04:14Z","failed_at":"e2e","signature_id":"ca52d27568368e0703bdb7fcdb2e8b54933f66f559908b6d0a48d5691bc53d6b","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xd266b380078\u003e: \n resource group \"patch-tags-6wvvmpdg8f9c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-6wvvmpdg8f9c\" to finish deleting: GET https://rp.j4430336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1803d96-bd6c-40cd-936a-45f1f9c9c199\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1803d96-bd6c-40cd-936a-45f1f9c9c199\",\n \"name\": \"f1803d96-bd6c-40cd-936a-45f1f9c9c199\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:10:26.245027108Z\",\n \"endTime\": \"2026-10-09T03:34:35.549070453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0kpdkn33trftvfg3ohnmt2hf90ta7 still exists (deletion dispatched at 2026-10-09T03:10:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xd266b380078\u003e: \n resource group \"patch-tags-6wvvmpdg8f9c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-6wvvmpdg8f9c\" to finish deleting: GET https://rp.j4430336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1803d96-bd6c-40cd-936a-45f1f9c9c199\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f1803d96-bd6c-40cd-936a-45f1f9c9c199\",\n \"name\": \"f1803d96-bd6c-40cd-936a-45f1f9c9c199\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:10:26.245027108Z\",\n \"endTime\": \"2026-10-09T03:34:35.549070453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0kpdkn33trftvfg3ohnmt2hf90ta7 still exists (deletion dispatched at 2026-10-09T03:10:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"97244f4abeebc52c84a7b3ebdedb2ed3d113900925ea1911d760956e3b024085","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377968924430336","occurred_at":"2026-10-09T02:04:14Z","failed_at":"e2e","signature_id":"dd5e9783b1066a92e3ba3b8da9f3bde91680259100a31f4f4dc4745788500224","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2741419abce0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2741419abce0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"ed50bfbd5500f57952a5aecd967cb6b4ea733a5a38e1851d38da023f00e67c8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377968924430336","occurred_at":"2026-10-09T02:04:14Z","failed_at":"e2e","signature_id":"c8ae875f9675b060f7ff6177afd27d7e1577f85c03248d7c33bede62663dad07","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2bebdca64108\u003e: \n resource group \"customer-rg-n6b8nxvh5gzl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-n6b8nxvh5gzl\" to finish deleting: GET https://rp.j4430336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b47eee2-4558-47ec-876a-e910a64edcab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b47eee2-4558-47ec-876a-e910a64edcab\",\n \"name\": \"0b47eee2-4558-47ec-876a-e910a64edcab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:09:52.911264338Z\",\n \"endTime\": \"2026-10-09T03:33:54.399258177Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2bebdca64108\u003e: \n resource group \"customer-rg-n6b8nxvh5gzl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-n6b8nxvh5gzl\" to finish deleting: GET https://rp.j4430336.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b47eee2-4558-47ec-876a-e910a64edcab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b47eee2-4558-47ec-876a-e910a64edcab\",\n \"name\": \"0b47eee2-4558-47ec-876a-e910a64edcab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:09:52.911264338Z\",\n \"endTime\": \"2026-10-09T03:33:54.399258177Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377959629852672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7180,"pr_state":"closed","pr_sha":"f58afcc1c7298b96467ad2dba099d8a155f0e337","final_merged_sha":"f58afcc1c7298b96467ad2dba099d8a155f0e337","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T02:04:11Z","started_at":"2026-10-09T02:04:11Z","completed_at":"2026-10-09T04:42:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T05:02:22.916111028Z","timing_metadata_checked_at":"2026-10-09T05:02:22.916111028Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T05:02:22.922361008Z","region_metadata_checked_at":"2026-10-09T05:02:22.922361008Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377922371850240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7292,"pr_state":"open","pr_sha":"e2249e5ba4dbcc59216d7341da5783394710eb68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T02:04:03Z","started_at":"2026-10-09T02:04:03Z","completed_at":"2026-10-09T04:32:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T04:51:42.494800409Z","timing_metadata_checked_at":"2026-10-09T04:51:42.494800409Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T04:51:42.502129486Z","region_metadata_checked_at":"2026-10-09T04:51:42.502129486Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8bcd5acdee294331a91928c33376a6f2c9aa477e486b5426dc8154f523ebe13c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377922371850240","occurred_at":"2026-10-09T02:04:03Z","failed_at":"e2e","signature_id":"c1431ac14fd0271bd4bc507228550d48034fc8fe91d5f2acb2d3e2d60688b09f","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:257]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullc5dl28vw558j.azurecr.io\nUnexpected error:\n \u003c*fmt.wrapError | 0x34f9246db800\u003e: \n VerifyImagePulled(acrpullc5dl28vw558j.azurecr.io/ubi-minimal) timed out after 2m0.002s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyImagePulled(acrpullc5dl28vw558j.azurecr.io/ubi-minimal) timed out after 2m0.002s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x34f9246db6e0\u003e{\n msg: \"failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x34f9246db5e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:257]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullc5dl28vw558j.azurecr.io\nUnexpected error:\n \u003c*fmt.wrapError | 0x34f9246db800\u003e: \n VerifyImagePulled(acrpullc5dl28vw558j.azurecr.io/ubi-minimal) timed out after 2m0.002s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyImagePulled(acrpullc5dl28vw558j.azurecr.io/ubi-minimal) timed out after 2m0.002s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x34f9246db6e0\u003e{\n msg: \"failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x34f9246db5e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"e3c9e2bb92a2c36dd413b60351c64bfd18acaa27312bc2a4a535e33137438c54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377922371850240","occurred_at":"2026-10-09T02:04:03Z","failed_at":"e2e","signature_id":"1e52c4ea4ce6b7b96618e329847bb0c9ab97f0ce6ec5f2a8a3add173085fac00","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x19485d7e23a0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-h78g452l2k2m\" to finish creating: GET https://rp.j1850240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2445641e-62af-46ed-8294-3ebf3cab568c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2445641e-62af-46ed-8294-3ebf3cab568c\",\n \"name\": \"2445641e-62af-46ed-8294-3ebf3cab568c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:36:09.491861222Z\",\n \"endTime\": \"2026-10-09T04:05:13.503264505Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t9g4t9s9x3s5g2e-np-delete-main-wlzrv-h29dk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-h78g452l2k2m--managed/providers/Microsoft.Compute/virtualMachines/t9g4t9s9x3s5g2e-np-delete-main-wlzrv-h29dk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t9g4t9s9x3s5g2e-np-delete-main-wlzrv-h29dk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-h78g452l2k2m--managed/providers/Microsoft.Compute/virtualMachines/t9g4t9s9x3s5g2e-np-delete-main-wlzrv-h29dk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x19485d7e23a0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-h78g452l2k2m\" to finish creating: GET https://rp.j1850240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2445641e-62af-46ed-8294-3ebf3cab568c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2445641e-62af-46ed-8294-3ebf3cab568c\",\n \"name\": \"2445641e-62af-46ed-8294-3ebf3cab568c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:36:09.491861222Z\",\n \"endTime\": \"2026-10-09T04:05:13.503264505Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t9g4t9s9x3s5g2e-np-delete-main-wlzrv-h29dk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-h78g452l2k2m--managed/providers/Microsoft.Compute/virtualMachines/t9g4t9s9x3s5g2e-np-delete-main-wlzrv-h29dk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t9g4t9s9x3s5g2e-np-delete-main-wlzrv-h29dk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-h78g452l2k2m--managed/providers/Microsoft.Compute/virtualMachines/t9g4t9s9x3s5g2e-np-delete-main-wlzrv-h29dk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-h78g452l2k2m\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377905737240576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7364,"pr_state":"closed","pr_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","final_merged_sha":"f10f5b479ee55c2f387bd6038920206e565c5c03","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T02:03:59Z","started_at":"2026-10-09T02:03:59Z","completed_at":"2026-10-09T04:52:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T05:12:40.122600471Z","timing_metadata_checked_at":"2026-10-09T05:12:40.122600471Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T05:12:40.125984961Z","region_metadata_checked_at":"2026-10-09T05:12:40.125984961Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a848f05747741a034a6a3452a67b5eefcc1da9a1d7be45c181e82d06e347724f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108377905737240576","occurred_at":"2026-10-09T02:03:59Z","failed_at":"e2e","signature_id":"6d149adf9696efaf8354e5d9b60fb40f03481cb87d334b82a607b6fd552d916b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x239dd5bc9700\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-sdfdh6slg4p7\" to finish creating: GET https://rp.j7240576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd44afef-8e0c-422b-a28f-1a1fdf5406e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd44afef-8e0c-422b-a28f-1a1fdf5406e3\",\n \"name\": \"cd44afef-8e0c-422b-a28f-1a1fdf5406e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:04:16.687507434Z\",\n \"endTime\": \"2026-10-09T03:33:25.201409198Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q7c4s8i0s9p3s5q-np-1-bvbvq-h94h9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-sdfdh6slg4p7--managed/providers/Microsoft.Compute/virtualMachines/q7c4s8i0s9p3s5q-np-1-bvbvq-h94h9 to exist\\nMachine q7c4s8i0s9p3s5q-np-1-bvbvq-vjj64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-sdfdh6slg4p7--managed/providers/Microsoft.Compute/virtualMachines/q7c4s8i0s9p3s5q-np-1-bvbvq-vjj64 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q7c4s8i0s9p3s5q-np-1-bvbvq-h94h9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-sdfdh6slg4p7--managed/providers/Microsoft.Compute/virtualMachines/q7c4s8i0s9p3s5q-np-1-bvbvq-h94h9 to exist\\nMachine q7c4s8i0s9p3s5q-np-1-bvbvq-vjj64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-sdfdh6slg4p7--managed/providers/Microsoft.Compute/virtualMachines/q7c4s8i0s9p3s5q-np-1-bvbvq-vjj64 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x239dd5bc9700\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-sdfdh6slg4p7\" to finish creating: GET https://rp.j7240576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd44afef-8e0c-422b-a28f-1a1fdf5406e3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd44afef-8e0c-422b-a28f-1a1fdf5406e3\",\n \"name\": \"cd44afef-8e0c-422b-a28f-1a1fdf5406e3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:04:16.687507434Z\",\n \"endTime\": \"2026-10-09T03:33:25.201409198Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q7c4s8i0s9p3s5q-np-1-bvbvq-h94h9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-sdfdh6slg4p7--managed/providers/Microsoft.Compute/virtualMachines/q7c4s8i0s9p3s5q-np-1-bvbvq-h94h9 to exist\\nMachine q7c4s8i0s9p3s5q-np-1-bvbvq-vjj64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-sdfdh6slg4p7--managed/providers/Microsoft.Compute/virtualMachines/q7c4s8i0s9p3s5q-np-1-bvbvq-vjj64 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q7c4s8i0s9p3s5q-np-1-bvbvq-h94h9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-sdfdh6slg4p7--managed/providers/Microsoft.Compute/virtualMachines/q7c4s8i0s9p3s5q-np-1-bvbvq-h94h9 to exist\\nMachine q7c4s8i0s9p3s5q-np-1-bvbvq-vjj64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-sdfdh6slg4p7--managed/providers/Microsoft.Compute/virtualMachines/q7c4s8i0s9p3s5q-np-1-bvbvq-vjj64 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7437,"pr_state":"closed","pr_sha":"6230ee414e6d12f612df8e8ae3353fe9e880c65a","final_merged_sha":"7495c811b9a906fee2419ebd6d9df2930140dd35","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-09T01:49:46Z","started_at":"2026-10-09T01:49:46Z","completed_at":"2026-10-09T04:14:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T04:20:42.737108686Z","timing_metadata_checked_at":"2026-10-09T04:20:42.737108686Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T04:20:42.744046564Z","region_metadata_checked_at":"2026-10-09T04:20:42.744046564Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"1354f2050975b59272f4312661b0defb924dba741805eab885f47576db7f575a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","failed_at":"e2e","signature_id":"617a9de46bb1c7108902990d2a726a9e7ea6051560c6bfa5ada743c1192b3686","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x4a16d2b40f0\u003e: \n resource group \"rg-candidate-5-1-trdpp8hx8tpd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-jjf49v\" in resourcegroup=\"rg-candidate-5-1-trdpp8hx8tpd\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/292a5ec0-f28b-48a4-89bc-34c6f69d9a46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/292a5ec0-f28b-48a4-89bc-34c6f69d9a46\",\n \"name\": \"292a5ec0-f28b-48a4-89bc-34c6f69d9a46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:05:28.202107106Z\",\n \"endTime\": \"2026-10-09T03:29:29.34914283Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0dh1i1h74jbtnnmtcppkg9ambqd8h still exists (deletion dispatched at 2026-10-09T03:05:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x4a16d2b40f0\u003e: \n resource group \"rg-candidate-5-1-trdpp8hx8tpd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-jjf49v\" in resourcegroup=\"rg-candidate-5-1-trdpp8hx8tpd\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/292a5ec0-f28b-48a4-89bc-34c6f69d9a46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/292a5ec0-f28b-48a4-89bc-34c6f69d9a46\",\n \"name\": \"292a5ec0-f28b-48a4-89bc-34c6f69d9a46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:05:28.202107106Z\",\n \"endTime\": \"2026-10-09T03:29:29.34914283Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0dh1i1h74jbtnnmtcppkg9ambqd8h still exists (deletion dispatched at 2026-10-09T03:05:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2460ac51c955a420378482921cd86eeb819cef7cf98fd095ec4224edbb0d226c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","failed_at":"e2e","signature_id":"ed915b16f8b2b776750f739a5daa97de703f32658ba93ea9faf3a154c2ab7b11","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1086ab296168\u003e: \n resource group \"v20261001preview-cr9lplpfwvph\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-cr9lplpfwvph\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/10d85540-0575-4e0a-abac-3407d1548f97\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/10d85540-0575-4e0a-abac-3407d1548f97\",\n \"name\": \"10d85540-0575-4e0a-abac-3407d1548f97\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:04:56.263351094Z\",\n \"endTime\": \"2026-10-09T03:28:59.770735963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0dh6ok3afvcebiteja6v1aqvho0u6 still exists (deletion dispatched at 2026-10-09T03:04:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1086ab296168\u003e: \n resource group \"v20261001preview-cr9lplpfwvph\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-cr9lplpfwvph\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/10d85540-0575-4e0a-abac-3407d1548f97\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/10d85540-0575-4e0a-abac-3407d1548f97\",\n \"name\": \"10d85540-0575-4e0a-abac-3407d1548f97\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:04:56.263351094Z\",\n \"endTime\": \"2026-10-09T03:28:59.770735963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0dh6ok3afvcebiteja6v1aqvho0u6 still exists (deletion dispatched at 2026-10-09T03:04:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2692710fa6f8d95920ed898d270ab2c1954816a04b0e2fb4023761886723eed3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","failed_at":"e2e","signature_id":"8249b8d9f8856d073b9d1060e96a79d6fd51a11374782dee26cbd35f7ce25db2","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3e5591194198\u003e: \n resource group \"rg-hs-pre--zqt4j66rqwpg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-84qkld\" in resourcegroup=\"rg-hs-pre--zqt4j66rqwpg\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aabc61f6-8a67-488f-a48c-a579beb22f14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aabc61f6-8a67-488f-a48c-a579beb22f14\",\n \"name\": \"aabc61f6-8a67-488f-a48c-a579beb22f14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:02:21.534993847Z\",\n \"endTime\": \"2026-10-09T03:26:29.648167314Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0dk6mkvl6568flpu0m9h3go953hh5 still exists (deletion dispatched at 2026-10-09T03:02:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3e5591194198\u003e: \n resource group \"rg-hs-pre--zqt4j66rqwpg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-84qkld\" in resourcegroup=\"rg-hs-pre--zqt4j66rqwpg\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aabc61f6-8a67-488f-a48c-a579beb22f14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aabc61f6-8a67-488f-a48c-a579beb22f14\",\n \"name\": \"aabc61f6-8a67-488f-a48c-a579beb22f14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:02:21.534993847Z\",\n \"endTime\": \"2026-10-09T03:26:29.648167314Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tb0dk6mkvl6568flpu0m9h3go953hh5 still exists (deletion dispatched at 2026-10-09T03:02:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9ebd9dd49f3ff99b14cba3661c428a14eb74506297f6d7b1241245d0693dd831","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","failed_at":"e2e","signature_id":"9b3b2f939bd60c04510443c5b5a36de01dbb0d75878b8f7c5c390016031c92b1","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:456]: error for disabled boot diagnostics should contain 409 status code\nExpected\n \u003cstring\u003e: failed to send request: Get \"https://admin.j1851392.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-mx7j6swwr25v/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/sre-hcp-cluster-bootdiag/serialconsole?vmName=d3i6t7c9q4h0k8j-worker-wrlpp-npg85\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\nto contain substring\n \u003cstring\u003e: 409\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:456]: error for disabled boot diagnostics should contain 409 status code\nExpected\n \u003cstring\u003e: failed to send request: Get \"https://admin.j1851392.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-mx7j6swwr25v/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/sre-hcp-cluster-bootdiag/serialconsole?vmName=d3i6t7c9q4h0k8j-worker-wrlpp-npg85\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\nto contain substring\n \u003cstring\u003e: 409","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ee2dc1e29d2efb4e37488a71b222c2bb43f2cba83c52cdf5162c1fd0f69c529","failure_pattern":"error for disabled boot diagnostics should contain 409 status code","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:456]: error for disabled boot diagnostics should contain 409 status code"}},{"row_id":"af5f44cc15579ff88384a3388fca0149f2d65b579b31fec761fe7a4b32db261d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","failed_at":"e2e","signature_id":"f5a624ce9e80c1ff18f16e6ce00681fccca997dd1474f86b90ce8aa6c5f750b3","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc360c750060\u003e: \n resource group \"rg-cp-ystream-upgrade-4-22-gx4vltwqcd64\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-pgmlq6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gx4vltwqcd64\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddf87703-cd9c-4434-ab6b-2db7a064f4fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddf87703-cd9c-4434-ab6b-2db7a064f4fc\",\n \"name\": \"ddf87703-cd9c-4434-ab6b-2db7a064f4fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:09:13.799569988Z\",\n \"endTime\": \"2026-10-09T03:33:19.091161643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc360c750060\u003e: \n resource group \"rg-cp-ystream-upgrade-4-22-gx4vltwqcd64\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-pgmlq6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-gx4vltwqcd64\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddf87703-cd9c-4434-ab6b-2db7a064f4fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ddf87703-cd9c-4434-ab6b-2db7a064f4fc\",\n \"name\": \"ddf87703-cd9c-4434-ab6b-2db7a064f4fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:09:13.799569988Z\",\n \"endTime\": \"2026-10-09T03:33:19.091161643Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bdf51fa058d985b585055abffc3c4a57862afe55caadf441d95bfce27ffc50f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","failed_at":"e2e","signature_id":"d18215fee9182d6fd0dffacde080da49fadf92738df8b9767a5a7d2cf9cc4833","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x124d3091c198\u003e: \n resource group \"rg-candidate-5-0-wjcmd5gpjktw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-q7hvjk\" in resourcegroup=\"rg-candidate-5-0-wjcmd5gpjktw\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/323be21f-0ba0-4b77-9aca-6bab469f3659\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/323be21f-0ba0-4b77-9aca-6bab469f3659\",\n \"name\": \"323be21f-0ba0-4b77-9aca-6bab469f3659\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:05:03.476461089Z\",\n \"endTime\": \"2026-10-09T03:29:09.085409713Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x124d3091c198\u003e: \n resource group \"rg-candidate-5-0-wjcmd5gpjktw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-q7hvjk\" in resourcegroup=\"rg-candidate-5-0-wjcmd5gpjktw\" to finish deleting: GET https://rp.j1851392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/323be21f-0ba0-4b77-9aca-6bab469f3659\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/323be21f-0ba0-4b77-9aca-6bab469f3659\",\n \"name\": \"323be21f-0ba0-4b77-9aca-6bab469f3659\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T03:05:03.476461089Z\",\n \"endTime\": \"2026-10-09T03:29:09.085409713Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c675f20c4ad90e9bdd60d7d7d3f643c19e0101ea2acb9e435f433b2a99183bad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-09T02:59:09Z\n Ended: 2026-10-09T03:12:06Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1851392-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"d4528e023dda36e13bcda68419aec7b30f31ea56b8c06ef89453053c6b6ef792","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7437/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108374328641851392","occurred_at":"2026-10-09T01:49:46Z","failed_at":"e2e","signature_id":"f23553b8fe3543a67ee1e3d8aafdef51bf55d49055f856784748065ac9884238","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:257]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullctpjh66q2dr2.azurecr.io\nUnexpected error:\n \u003c*fmt.wrapError | 0x97759ef0500\u003e: \n VerifyImagePulled(acrpullctpjh66q2dr2.azurecr.io/ubi-minimal) timed out after 2m0.001s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyImagePulled(acrpullctpjh66q2dr2.azurecr.io/ubi-minimal) timed out after 2m0.001s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x97759ef04a0\u003e{\n msg: \"failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x97759f21ec0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:257]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullctpjh66q2dr2.azurecr.io\nUnexpected error:\n \u003c*fmt.wrapError | 0x97759ef0500\u003e: \n VerifyImagePulled(acrpullctpjh66q2dr2.azurecr.io/ubi-minimal) timed out after 2m0.001s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyImagePulled(acrpullctpjh66q2dr2.azurecr.io/ubi-minimal) timed out after 2m0.001s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x97759ef04a0\u003e{\n msg: \"failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x97759f21ec0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["1a6d95c4937e5c083ddbb2144ddffd0453da18207fcc58ddb599e5f021ffa0ec","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","8ee2dc1e29d2efb4e37488a71b222c2bb43f2cba83c52cdf5162c1fd0f69c529","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108368505299537920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"1fc9a8ec2d2a2ede4d690afbd9a0636fb7ec01fd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-09T01:26:37Z","started_at":"2026-10-09T01:26:37Z","completed_at":"2026-10-09T03:54:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T04:10:07.150957479Z","timing_metadata_checked_at":"2026-10-09T04:10:07.150957479Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T04:10:07.157323858Z","region_metadata_checked_at":"2026-10-09T04:10:07.157323858Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":9,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":2,"runs_without_occurrence_rows":5,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T23:00:53Z","started_at":"2026-10-09T23:00:53Z","completed_at":"2026-10-10T00:51:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T01:00:05.965737009Z","timing_metadata_checked_at":"2026-10-10T01:00:05.965737009Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T01:00:05.97844917Z","region_metadata_checked_at":"2026-10-10T01:00:05.97844917Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"102b2d8a5e60bac19037e3422d10def66a711a5b0ab8a307c35de756221eb153","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"943567e0e7942ccbf9ee1f42758dbeafd6a415e8fd929d6f9edd74a6a92d458e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x284ab7439230\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-4qsn8gb8p28c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x284ab7439230\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-4qsn8gb8p28c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"56ec8d19005a19692d2ae939b295f7610da1e96240bdc71fd31d597e6e2b8841","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"028fb357f573ffa054d5144583f4c22cd50b81ab699f7eee531ee4cdeb8f5a03","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2a25db7081b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gkw6wzhpx84w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2a25db7081b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gkw6wzhpx84w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5eb34d59d86dd0579981ef5b85e534a3448bfd148f7d2d720cabbbe209b9f13a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"c3a144c1a4b30ab1c9c2b6f88c994e5695f256aa071d503e64b63c37d33f0d7d","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3f756cdcc8d0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-qq6g9wz95bjw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3f756cdcc8d0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-qq6g9wz95bjw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e034da95e3fdac4375a93652758dc49bb09b11fe686f0b5ef5ecd76325e3d596","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"5f2ffe9a21529b96388c2140816f5b164f7b9b0de07aecad7e22bbca275aaa63","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x30713b85be60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-tffs96fmr5p6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x30713b85be60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-tffs96fmr5p6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e2afb02d8f8ab77c948ca2b80555ee8e919ae4ce18f09b12d143c272ccd21d77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108694214656135168","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"b6f4654eb69ad45b39e2c965dcb1bb7725b86526f162a602d4de14c605a4bbec","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x25bb28101bc0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bbmhspx64hh7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x25bb28101bc0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bbmhspx64hh7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108557920571494400","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T13:59:15Z","started_at":"2026-10-09T13:59:15Z","completed_at":"2026-10-09T16:38:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T16:43:30.153176654Z","timing_metadata_checked_at":"2026-10-09T16:43:30.153176654Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T16:43:30.155986645Z","region_metadata_checked_at":"2026-10-09T16:43:30.155986645Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"853640d8edf412d6f54c5ecec4b8a87d0e0d1ef4fb23d9feea2f142db5f26f94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108557920571494400","occurred_at":"2026-10-09T13:59:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108557181132476416","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T13:56:21Z","started_at":"2026-10-09T13:56:21Z","completed_at":"2026-10-09T16:34:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T16:43:29.645897821Z","timing_metadata_checked_at":"2026-10-09T16:43:29.645897821Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T16:43:29.650468506Z","region_metadata_checked_at":"2026-10-09T16:43:29.650468506Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108557135892713472","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T13:56:10Z","started_at":"2026-10-09T13:56:10Z","completed_at":"2026-10-09T16:41:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T16:43:29.385917124Z","timing_metadata_checked_at":"2026-10-09T16:43:29.385917124Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T16:43:29.389596913Z","region_metadata_checked_at":"2026-10-09T16:43:29.389596913Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108555842956562432","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T13:51:02Z","started_at":"2026-10-09T13:51:02Z","completed_at":"2026-10-09T16:40:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T16:43:29.189622331Z","timing_metadata_checked_at":"2026-10-09T16:43:29.189622331Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T16:43:29.193362619Z","region_metadata_checked_at":"2026-10-09T16:43:29.193362619Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108529529541627904","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T12:05:23Z","started_at":"2026-10-09T12:05:23Z","completed_at":"2026-10-09T14:37:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T14:58:38.466710974Z","timing_metadata_checked_at":"2026-10-09T14:58:38.466710974Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T14:58:38.470112263Z","region_metadata_checked_at":"2026-10-09T14:58:38.470112263Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108527023956692992","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T11:56:31Z","started_at":"2026-10-09T11:56:31Z","completed_at":"2026-10-09T11:59:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T12:20:52.317852988Z","timing_metadata_checked_at":"2026-10-09T12:20:52.317852988Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-09T12:20:52.325415564Z","region_metadata_checked_at":"2026-10-09T13:24:50.813343245Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"30743b29deefc1214fe47990297f897bdd1f11f3d10b627e82f2127651239160","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108527023956692992","occurred_at":"2026-10-09T11:56:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T02:00:38Z","started_at":"2026-10-09T02:00:38Z","completed_at":"2026-10-09T04:06:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T04:20:42.91503993Z","timing_metadata_checked_at":"2026-10-09T04:20:42.91503993Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T04:20:42.919826715Z","region_metadata_checked_at":"2026-10-09T04:20:42.919826715Z"},"failed_at":["e2e"],"failed_test_count":22,"occurrences":[{"row_id":"01cfcb8865e1e9e0b2fd75af4dab15399e058e9c8a7520cf14a979b37f5bec03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"ae372b6b9650d0b7fb76a731c6bb73e771d1ad2a2f403c8ea046a4810a89d911","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x15ccc8e3e260\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-s4lpkb\" resourcegroup=\"rg-np-version-upgrade-s4lpkb-nckb2tfdx9sw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-s4lpkb-nckb2tfdx9sw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-s4lpkb/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x15ccc8e3e260\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-s4lpkb\" resourcegroup=\"rg-np-version-upgrade-s4lpkb-nckb2tfdx9sw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-s4lpkb-nckb2tfdx9sw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-s4lpkb/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"0d8a70dbc64a000ab75060957d469e777c4087f2a9bc924ff4a067201f6f4e97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"e23504d5c07814913d080ea69d0ded698df5a4d99822500032ec3653c9766294","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2ec55b7f8360\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vqbwskk476qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x2ec55b7f8360\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-vqbwskk476qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"17e90f327e78c9f6c0eb1e799bb44736b7299747eeb845a9bff1c3760738a580","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"7a7082dfbd050591871fcb9c5978abf946cd65bce9ed32eda21da8403bff14ed","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-9h95dn\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0x1b712a5307b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-fb447pvfc5mf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9h95dn\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-9h95dn\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0x1b712a5307b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-fb447pvfc5mf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9h95dn\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"20cf368795bcb8b9f186c080fc61b020e9e61683bf09d818c1a5d851b0647f45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"78ec029001c30998a4e1babdab79d48c3c17f2f252beb1cc1f2638c1cfdd5957","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-7pp5zf\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x3221c9e003f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dhsqvc8s4dts/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-7pp5zf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-7pp5zf\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x3221c9e003f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dhsqvc8s4dts/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-7pp5zf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"24310f5d1081a4f0d0ef8a704796ca0f6497f2526a08ca2d8a9ae364479cf908","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"5091085de46b6155ec8a80aa3a08e418ed8e2e45274e5a57cc44080adfe9031d","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f4d1e67e100\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-5nlftxqfx9mj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-5nlftxqfx9mj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f4d1e67e100\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-5nlftxqfx9mj\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-5nlftxqfx9mj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"246c4acebcfc12b909cc133951b3a4ae249b8d3836f73affd262145e271dbeac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"4a276b7331de267a380c97189d1fcd50c4f0d29178edf855603c725af8fd356e","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0x22aa54096dc0\u003e: \n failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-qs6slk\" resourcegroup=\"rg-np-skip-minor-qs6slk-svmgzjwq29hn\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-qs6slk-svmgzjwq29hn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-qs6slk/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0x22aa54096dc0\u003e: \n failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-qs6slk\" resourcegroup=\"rg-np-skip-minor-qs6slk-svmgzjwq29hn\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-qs6slk-svmgzjwq29hn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-qs6slk/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"253e99a84eaae36680a37b8aa416a3086748aa7e0fe0c2ac92e9a8e6d0a7e225","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"22c5f8ad3e9edbc947e95262b2baf90209d9181c5fe0a567570ea7382201e9a4","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x6f0cf085080\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-6zf46c\" resourcegroup=\"rg-np-dg-minor-6zf46c-hll6hfr44s9d\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-6zf46c-hll6hfr44s9d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-6zf46c/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x6f0cf085080\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-6zf46c\" resourcegroup=\"rg-np-dg-minor-6zf46c-hll6hfr44s9d\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-6zf46c-hll6hfr44s9d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-6zf46c/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"270c242efb5c958572b1997feac0b8ed5f2e345267a47589878fc3f78255a78e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"f3603a0b53eed9bd4dff440d7ea05c63ed18b5ef29f36ef1fb22c8746fbc2725","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0x1663af28c900\u003e: \n failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-qlgw5r\" resourcegroup=\"rg-np-downgrade-qlgw5r-fvqqvhgtclmz\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-qlgw5r-fvqqvhgtclmz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-qlgw5r/nodePools/npdg-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0x1663af28c900\u003e: \n failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-qlgw5r\" resourcegroup=\"rg-np-downgrade-qlgw5r-fvqqvhgtclmz\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-qlgw5r-fvqqvhgtclmz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-qlgw5r/nodePools/npdg-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"321ddf4681e8dbc3b8364d71796ad6ad6f8bedbb40c0bc2447e343a6930230e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"d4bf2a2be500d39b22377146543f50824f15b605e910ecb76b543ea51e8343e0","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-j44hdg\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xf1601f277d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-7nd955rpcdtq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-j44hdg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-j44hdg\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xf1601f277d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-7nd955rpcdtq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-j44hdg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"41749547f7158b7412f1198065073cb2db802aa15c4fc688b7064b4229432b9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"2f66d792165c52d205f1c6a611ab324b91f4405832fbf3e31394198c2a8e5ca9","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-s865cq\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x29e047bb9ce0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-84fl6x5gkgjr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-s865cq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-s865cq\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x29e047bb9ce0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-84fl6x5gkgjr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-s865cq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"47ea5f74535f8c2365c67e6e152bbf5a210e1ae70914858cdf0a316857badd23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"d0f6c4eb56b2416135b269c1aa7389148782dac649d937c3c1e7b19d1f13896e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x172168187dd0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ffsspp7xdxbw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x172168187dd0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-ffsspp7xdxbw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4c4c68639c776eb2e3a3c22919ddd840da35a56d0e4fa25f61acde0f87495c5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"957ef542fb913c7900eb599a22e4916d0e0fd5c25e4c722cbf8460aa0fd3dd33","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0xad5396d32a0\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-wl7vjdkvvpf7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-wl7vjdkvvpf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0xad5396d32a0\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-wl7vjdkvvpf7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-wl7vjdkvvpf7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"542369570470b39ac955310ea088d8f04bd5f813873200eaacb63cd645b771b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"d06c5ac4362d15796d9f2f41e12315787900a50fa301ee39c8a256c3850ceb32","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-kqzff9\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xe01aaa2bbc0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-p7pmn5ng5wrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-kqzff9\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-kqzff9\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xe01aaa2bbc0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-p7pmn5ng5wrg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-kqzff9\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"56501d5bde39c72f421e6677cf6441196b1866748973c00e46a9023c7e2d993a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"06438279994666e4b2d4e4cda8087f66299b5a578c7b2cd13360e3d1df435524","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-g5xfmb\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x2d844c21d830\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-5xdvnklmwfq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-g5xfmb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-g5xfmb\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x2d844c21d830\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-5xdvnklmwfq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-g5xfmb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6a79ffa0651c38d11a6e9e3958564321f5119f1937699cf99ccc563c45e2144c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"e39234445903bc24ac0fa7548e2bc935524d10f1a9dce0f385ccddebb248cdef","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x669cc1b95c0\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-mwkm86\" resourcegroup=\"rg-np-version-upgrade-mwkm86-hh468p9mpt76\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mwkm86-hh468p9mpt76/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mwkm86/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x669cc1b95c0\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-mwkm86\" resourcegroup=\"rg-np-version-upgrade-mwkm86-hh468p9mpt76\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mwkm86-hh468p9mpt76/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mwkm86/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a02ef24273b9de68f36b0b6aa9376060ac3028f4596488d84c97db08d0362aa6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"589b0f77a559ee7a26153a17c6bf39d2e9b99a08197510cd4eaee21d45be7bf3","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-qldg5l\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x200557060510\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-hldpsj9c8292/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-qldg5l\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-qldg5l\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x200557060510\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-hldpsj9c8292/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-qldg5l\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a3dc22e4804ca5b81ab135462520d253d7ceaf1413bed6d2023550cd7f63a2dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"34397fe5abf712908f5423e76c41e9fb9bf115f7d4c93ff88752ee98026f7fa9","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-pwmrlt\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x364694577140\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-bcxf7z6jvkdm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-pwmrlt\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-pwmrlt\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x364694577140\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-bcxf7z6jvkdm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-pwmrlt\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ccdcd9a6e94aed93353c31f506d4d2c990dac817db5493c6d493eef0e66877a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"7fed0e3251f5b0df1a0ed777ac58de15b61f4ae1193d04b925b5765a33b1cadd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-t4k2nv\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x9a9211d2ff0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-wdg9f8phhv6h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-t4k2nv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-t4k2nv\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x9a9211d2ff0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-wdg9f8phhv6h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-t4k2nv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"cdc5dd70104c26ef82228dbeeaa35e6ec78d84498d0f69cc08fa4a2d69df2467","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"3997c3e7b5d9bc914c8451d8312e28fb0c306a5f38d600fd0ce6f681823af251","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x1508ab4d7940\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-lc4jfj\" resourcegroup=\"rg-np-version-upgrade-lc4jfj-qb5cmdpk8ppz\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lc4jfj-qb5cmdpk8ppz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-lc4jfj/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x1508ab4d7940\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-lc4jfj\" resourcegroup=\"rg-np-version-upgrade-lc4jfj-qb5cmdpk8ppz\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lc4jfj-qb5cmdpk8ppz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-lc4jfj/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d381dfbc14e4941c9e094caa4be144b0069586daf9c381b31b0e4ecdcc79095c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"7b0aa1f88528b7aab6ebd43d0c8bc0c167b397681dde35e640fb0e5257495a48","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-9d5886\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x2e46520ddbc0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-k6tc5l862fq5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-9d5886\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-9d5886\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x2e46520ddbc0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-k6tc5l862fq5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-9d5886\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"db77486c06f5aeca0109d5f9c8292906e6c0978739b5f3ecae40da6ec9b67429","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"da1b16a2e6b7eedee000ee0af62d7085a036babbf6f8b626eaec2f300a3d2e39","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2554f7d0b4d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-6xchsw5qtmxd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2554f7d0b4d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-6xchsw5qtmxd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f413c672286505afbd34eda37755f0a7db7987891c1e92302b5026d806e46062","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108377065311965184","occurred_at":"2026-10-09T02:00:38Z","failed_at":"e2e","signature_id":"ab1b252cdf5012fbe93f692e1802a7010b7b1c0d6094b73f85e4facc9115996a","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x18577971d620\u003e: \n failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-j47stk\" resourcegroup=\"rg-np-noedge-j47stk-c7scxw66l4hg\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-j47stk-c7scxw66l4hg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-j47stk/nodePools/npnoedge-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x18577971d620\u003e: \n failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-j47stk\" resourcegroup=\"rg-np-noedge-j47stk-c7scxw66l4hg\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-j47stk-c7scxw66l4hg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-j47stk/nodePools/npnoedge-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":22,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610"],"matched_occurrences":22,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108357557067190272","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T00:43:07Z","started_at":"2026-10-09T00:43:07Z","completed_at":"2026-10-09T03:21:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T03:38:48.617416077Z","timing_metadata_checked_at":"2026-10-09T03:38:48.617416077Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T03:38:48.625467151Z","region_metadata_checked_at":"2026-10-09T03:38:48.625467151Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T23:00:53Z","started_at":"2026-10-09T23:00:53Z","completed_at":"2026-10-10T00:35:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-10T00:49:45.706196251Z","timing_metadata_checked_at":"2026-10-10T00:49:45.706196251Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-10T00:49:45.709498342Z","region_metadata_checked_at":"2026-10-10T00:49:45.709498342Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"36e59c21f026f3cf5181906219c0d58ae52660a53d5bb8e389cd1fdad256727e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"4eca13a165c2ccc73b835fae934265c145306a5b3c4d9ed3f076311aed85c4b1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3fc6670d91a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mgmsfgq99bg9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3fc6670d91a0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mgmsfgq99bg9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5f8d5ce9bbfbdfb4c1d5523f4c7fa13414f93695ff3a86122f3dfd79bdab2c95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"d81d7f06c05c906562011f64261d02893fd0c274bf91ed65c6010a47e11dd46c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x3ecd88409b60\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fjsmwtbqw48w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x3ecd88409b60\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-fjsmwtbqw48w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"836ac9190628ff984949dc58eca265e8709d5f4f9f2dcd7cf36acb221408c555","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"daeed2ecbf508b35f1ba38005d11ff1479283bbc484b275758e91ca9f68a2891","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x179ee7733d70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-nxkg9dgmqghf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x179ee7733d70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-nxkg9dgmqghf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bb92497fe112cae88dfa018d4af4c2c5f5b33110735e98c43f560045d192456c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"500fd5ba3e339ff6a02a8c3df741232041d04709e81cedd8b10bf9374ceb0e4d","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3f9b94218c90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-jr65q449l4wp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x3f9b94218c90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-jr65q449l4wp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d01441bde8cb4f52e526bf56c74f2767079af9beb80e9e7bd3a4049919f0b091","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2108694214664523776","occurred_at":"2026-10-09T23:00:53Z","failed_at":"e2e","signature_id":"3c386237da479ec8664fa6d9ac7a9bd2aa5df117f4040d153a6a01fd3df4918e","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x16ef84387050\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8hwhxx82br9m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x16ef84387050\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8hwhxx82br9m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2108601007108788224","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T16:50:30Z","started_at":"2026-10-09T16:50:30Z","completed_at":"2026-10-09T19:07:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T19:26:23.734893135Z","timing_metadata_checked_at":"2026-10-09T19:26:23.734893135Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T19:26:23.738912322Z","region_metadata_checked_at":"2026-10-09T19:26:23.738912322Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2108427018348531712","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-09T05:19:08Z","started_at":"2026-10-09T05:19:08Z","completed_at":"2026-10-09T07:52:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T08:10:29.664568161Z","timing_metadata_checked_at":"2026-10-09T08:10:29.664568161Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T08:10:29.669517845Z","region_metadata_checked_at":"2026-10-09T08:10:29.669517845Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bd2a3665f2db3ec46c7605a8f00a537fceb62b99114d34846dbc8b969e3a8d0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2108427018348531712","occurred_at":"2026-10-09T05:19:08Z","failed_at":"e2e","signature_id":"688238fa4fec4ee5a50f4498249bef5708c8fb182d2ba3d5ee6a9bc11ef48c84","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x13cc8f58b420\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-sz5gdnfzqhdd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c72bb6d-878b-457e-bf39-56f98851abda\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c72bb6d-878b-457e-bf39-56f98851abda\",\n \"name\": \"2c72bb6d-878b-457e-bf39-56f98851abda\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T06:39:07.400427011Z\",\n \"endTime\": \"2026-10-09T07:08:09.774067703Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-sz5gdnfzqhdd--managed/providers/Microsoft.Compute/virtualMachines/y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-sz5gdnfzqhdd--managed/providers/Microsoft.Compute/virtualMachines/y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x13cc8f58b420\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-sz5gdnfzqhdd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c72bb6d-878b-457e-bf39-56f98851abda\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c72bb6d-878b-457e-bf39-56f98851abda\",\n \"name\": \"2c72bb6d-878b-457e-bf39-56f98851abda\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-09T06:39:07.400427011Z\",\n \"endTime\": \"2026-10-09T07:08:09.774067703Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-sz5gdnfzqhdd--managed/providers/Microsoft.Compute/virtualMachines/y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-sz5gdnfzqhdd--managed/providers/Microsoft.Compute/virtualMachines/y1p1d9s3x8k4l0j-cordon-np-ht298-w2dcd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2108377065324548096","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-09T02:00:38Z","started_at":"2026-10-09T02:00:38Z","completed_at":"2026-10-09T04:05:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T04:20:43.05936248Z","timing_metadata_checked_at":"2026-10-09T04:20:43.05936248Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T04:20:43.065562961Z","region_metadata_checked_at":"2026-10-09T04:20:43.065562961Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}