{"meta":{"date":"2026-10-16","timezone":"UTC","generated_at":"2026-11-01T07:44:55Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":75,"failed_runs":48,"runs_with_occurrences":48,"runs_with_failure_pattern_matches":48,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":27,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111233385828454400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7580,"pr_state":"closed","pr_sha":"0c611949a02927c684b102343dcccc0663b92a88","final_merged_sha":"cc68e5f22f90cd600ec1702fd6051be82c4f0a33","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T23:10:38Z","started_at":"2026-10-16T23:10:38Z","completed_at":"2026-10-16T23:25:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T23:42:46.579558997Z","timing_metadata_checked_at":"2026-10-16T23:42:46.579558997Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T23:42:46.583578084Z","region_metadata_checked_at":"2026-10-16T23:42:46.583578084Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"59eef0498945003528e24fa262b73d8903679a90474c5f0cab2f027f7a345dbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111233385828454400","occurred_at":"2026-10-16T23:10:38Z","failed_at":"provision","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T23:16:01.424Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T23:16:02.804Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc","failure_pattern":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found","search_query":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111228423002132480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"d319c1d4067027f832f7272e4bfccc3580171943","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T22:50:55Z","started_at":"2026-10-16T22:50:55Z","completed_at":"2026-10-16T23:32:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T23:42:46.390326504Z","timing_metadata_checked_at":"2026-10-16T23:42:46.390326504Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T23:42:46.394437291Z","region_metadata_checked_at":"2026-10-16T23:42:46.394437291Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bd4443d05f2912c939d3f8dc77041dffbcce3e1c5bfb6fef45da617340190587","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111228423002132480","occurred_at":"2026-10-16T22:50:55Z","failed_at":"provision","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T23:22:36.468Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-16T23:22:36.986Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]\"\ntime=2026-10-16T23:22:36.986Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-16T23:22:37.251Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-16T23:22:37.674Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-16T23:22:37.674Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-16T23:22:37.803Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-16T23:23:44.090Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-16T23:23:44.090Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111224357266132992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"924de57c9ad6f6049da7964e3f0bebdf0cf7eb7a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T22:34:46Z","started_at":"2026-10-16T22:34:46Z","completed_at":"2026-10-16T23:35:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T23:53:07.30412143Z","timing_metadata_checked_at":"2026-10-16T23:53:07.30412143Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T23:53:07.307710419Z","region_metadata_checked_at":"2026-10-16T23:53:07.307710419Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a8779eaeea88183991fc6db0887e6e5931175101be3f18fb0bdf9c53683a1e0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111224357266132992","occurred_at":"2026-10-16T22:34:46Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T23:13:05.250Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-16T23:13:06.422Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-16T23:13:06.422Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-16T23:13:07.473Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-16T23:13:07.473Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6132992-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-16T23:13:07.473Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j6132992-svc\ntime=2026-10-16T23:28:10.126Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111219106693779456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7580,"pr_state":"closed","pr_sha":"0c611949a02927c684b102343dcccc0663b92a88","final_merged_sha":"cc68e5f22f90cd600ec1702fd6051be82c4f0a33","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T22:13:54Z","started_at":"2026-10-16T22:13:54Z","completed_at":"2026-10-16T22:45:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T23:01:12.922444696Z","timing_metadata_checked_at":"2026-10-16T23:01:12.922444696Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T23:01:12.926334683Z","region_metadata_checked_at":"2026-10-16T23:01:12.926334683Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1ebaa172322d32c151fc47afc25d46f422946a931d9f0527579f8a4f84edc192","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111219106693779456","occurred_at":"2026-10-16T22:13:54Z","failed_at":"provision","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T22:41:06.800Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T22:41:08.120Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc","failure_pattern":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found","search_query":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111218975433035776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7531,"pr_state":"closed","pr_sha":"7d46ec35677c4e23f31a3e4c9085956e5e727614","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T22:13:22Z","started_at":"2026-10-16T22:13:22Z","completed_at":"2026-10-17T00:39:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-17T00:56:24.686172491Z","timing_metadata_checked_at":"2026-10-17T00:56:24.686172491Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-17T00:56:24.690295377Z","region_metadata_checked_at":"2026-10-17T00:56:24.690295377Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"dd90f33260eddf465ed6facc16556fdbf16f51e502bcc6e99cb7d4c1204c58c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111218975433035776","occurred_at":"2026-10-16T22:13:22Z","failed_at":"e2e","signature_id":"232f7dd2673899fec5aeff9706e5509b4be07bc67cbc3af52825b2dc89f2cb5e","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x165441e2cf48\u003e: \n VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\n {\n errs: [\n \u003c*fmt.wrapError | 0x165441dab3a0\u003e{\n msg: \"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n err: \u003c*fmt.wrapError | 0x165441dab1c0\u003e{\n msg: \"connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n err: \u003c*errors.errorString | 0x1654427290b0\u003e{\n s: \"test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x165441e2cf48\u003e: \n VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\n {\n errs: [\n \u003c*fmt.wrapError | 0x165441dab3a0\u003e{\n msg: \"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n err: \u003c*fmt.wrapError | 0x165441dab1c0\u003e{\n msg: \"connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n err: \u003c*errors.errorString | 0x1654427290b0\u003e{\n s: \"test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]\",\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7da5fb7000522ac8f196f5fe9701e1ceef598ae9e4de9c38e95fe95ace38654e","failure_pattern":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff)","search_query":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-bhfzdb namespace are ready: test pods not yet ready: [pod-to-a-5b5d56bbc9-gjjkg (container pod-to-a-container waiting: CrashLoopBackOff) pod-to-a-allowed-cnp-ddf755dfc-vqspz (container pod-to-a-allowed-cnp-container waiting: CrashLoopBackOff) pod-to-b-multi-node-headless-8656ffdc6-6hrzt (container pod-to-b-multi-node-headless-container waiting: CrashLoopBackOff)]"}},{"row_id":"f2df4cf94cc0936c1b9ac2535505da0c1d8a434808bdee570a6e90aeb8f7a992","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111218975433035776","occurred_at":"2026-10-16T22:13:22Z","failed_at":"e2e","signature_id":"2bf01ccbacd9feffa73eddf299b329d411264d68e14c66906bd09f4e0f7f7b5d","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x33c730fe0780\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2q8nt57gw7c4\" to finish creating: GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\",\n \"name\": \"f4f78025-9771-4e07-9a76-cc316ad5bcaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T23:21:44.60401103Z\",\n \"endTime\": \"2026-10-16T23:40:48.245826795Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-2q8nt57gw7c4--managed/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Network/networkInterfaces/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\n correlation ID: 57a29177-39d2-4d3f-ac56-44b9805e57f9\n client request ID: 8065bb9a-6902-4747-a2b4-2914d4dc8e70\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x33c730fe0780\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2q8nt57gw7c4\" to finish creating: GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\",\n \"name\": \"f4f78025-9771-4e07-9a76-cc316ad5bcaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T23:21:44.60401103Z\",\n \"endTime\": \"2026-10-16T23:40:48.245826795Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-2q8nt57gw7c4--managed/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Network/networkInterfaces/p9v1y5n5j8r9i0x-np-1-89jk5-vpf5w-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\nMachine p9v1y5n5j8r9i0x-np-1-89jk5-zffrv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2q8nt57gw7c4--managed/providers/Microsoft.Compute/virtualMachines/p9v1y5n5j8r9i0x-np-1-89jk5-zffrv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3035776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f78025-9771-4e07-9a76-cc316ad5bcaa\n correlation ID: 57a29177-39d2-4d3f-ac56-44b9805e57f9\n client request ID: 8065bb9a-6902-4747-a2b4-2914d4dc8e70\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a35aeac2b02d0bb0965498ad4bf13cb371d32eb6b72e0ddbe98ebe03cb92ab3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer ... message truncated Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1a35aeac2b02d0bb0965498ad4bf13cb371d32eb6b72e0ddbe98ebe03cb92ab3","7da5fb7000522ac8f196f5fe9701e1ceef598ae9e4de9c38e95fe95ace38654e"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111186341810147328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7531,"pr_state":"closed","pr_sha":"7d46ec35677c4e23f31a3e4c9085956e5e727614","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T20:03:41Z","started_at":"2026-10-16T20:03:41Z","completed_at":"2026-10-16T22:10:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T22:20:04.674189634Z","timing_metadata_checked_at":"2026-10-16T22:20:04.674189634Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T22:20:04.699299251Z","region_metadata_checked_at":"2026-10-16T22:20:04.699299251Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8f0c841cbe717eb4c377950aabacea515c3394f8444b080019222a0690b67e1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111186341810147328","occurred_at":"2026-10-16T20:03:41Z","failed_at":"e2e","signature_id":"0270ee34b1d1960ce57816d9c23f2bf4723eabf324cf85fb883ca827cb1fc8d2","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x2fa3cd2a3c20\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-wl8xt5tlkcq7\" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\",\n \"name\": \"63825d69-d610-4421-b3ee-50ac654908c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T21:00:14.715888281Z\",\n \"endTime\": \"2026-10-16T21:19:17.454369198Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-rnlcf-ttbgr: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-wl8xt5tlkcq7--managed/ea-cluster-ea-np-1-rnlcf-ttbgr-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wl8xt5tlkcq7--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-rnlcf-ttbgr-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\n correlation ID: 2667b7c1-2456-4fb3-a1b3-9f090199ed55\n client request ID: 1483f922-ab7c-4902-933a-1a3aef1cf957\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x2fa3cd2a3c20\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-wl8xt5tlkcq7\" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\",\n \"name\": \"63825d69-d610-4421-b3ee-50ac654908c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T21:00:14.715888281Z\",\n \"endTime\": \"2026-10-16T21:19:17.454369198Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-rnlcf-ttbgr: Failed: interfaces failed to create or update. err: failed to get existing resource external-auth-cluster-wl8xt5tlkcq7--managed/ea-cluster-ea-np-1-rnlcf-ttbgr-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-wl8xt5tlkcq7--managed/providers/Microsoft.Network/networkInterfaces/ea-cluster-ea-np-1-rnlcf-ttbgr-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/63825d69-d610-4421-b3ee-50ac654908c5\n correlation ID: 2667b7c1-2456-4fb3-a1b3-9f090199ed55\n client request ID: 1483f922-ab7c-4902-933a-1a3aef1cf957\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3a08509cc4313fdb7de088f97578445c52be0514ee3ea35913c562d65fc01af4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"98ab4c7689f42008eeaecc35e095e2ae2f37054c2826f6b870431500aab5d0ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7531/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111186341810147328","occurred_at":"2026-10-16T20:03:41Z","failed_at":"e2e","signature_id":"05641d1f46b281713df86eeeae3a1d62541348ec25f684921bc566e261191e5e","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d04d19fc660\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-kjxkhz\" in resourcegroup=\"rg-np-version-upgrade-kjxkhz-ldkgrb6cvc64\" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\",\n \"name\": \"eec0983a-6d6f-417c-a350-df06c778abae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T21:16:56.042187584Z\",\n \"endTime\": \"2026-10-16T21:35:57.540237831Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\n correlation ID: 62969ebc-fdd1-4a1a-808f-bd02fcd158bd\n client request ID: f0c87d09-298f-426d-885a-00fac76d2bf1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d04d19fc660\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-kjxkhz\" in resourcegroup=\"rg-np-version-upgrade-kjxkhz-ldkgrb6cvc64\" to finish creating: GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\",\n \"name\": \"eec0983a-6d6f-417c-a350-df06c778abae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T21:16:56.042187584Z\",\n \"endTime\": \"2026-10-16T21:35:57.540237831Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o9t1p7r8v5x0r1j-npupgrade-4-20-jqtb8-n8bff: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0147328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eec0983a-6d6f-417c-a350-df06c778abae\n correlation ID: 62969ebc-fdd1-4a1a-808f-bd02fcd158bd\n client request ID: f0c87d09-298f-426d-885a-00fac76d2bf1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["3a08509cc4313fdb7de088f97578445c52be0514ee3ea35913c562d65fc01af4","f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111186249774534656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7483,"pr_state":"open","pr_sha":"03e9a9e34dff8fff4847f513c07fbd5a685545c5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T20:03:20Z","started_at":"2026-10-16T20:03:20Z","completed_at":"2026-10-16T22:12:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T22:30:17.812317132Z","timing_metadata_checked_at":"2026-10-16T22:30:17.812317132Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T22:30:17.816558818Z","region_metadata_checked_at":"2026-10-16T22:30:17.816558818Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"dbe90422091f720346376969333f094c7dfe6f6549f911dca5f39ed4e2b10533","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111186249774534656","occurred_at":"2026-10-16T20:03:20Z","failed_at":"alert","signature_id":"9e8e933175b0a2604bba19bae3f44ffb21292acf78887aa2195f8690115a15e3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T21:21:48Z\n Ended: 2026-10-16T21:29:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4534656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q\", pod=\"ignition-server-proxy-d589845df-77l2s\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q/ignition-server-proxy-d589845df-77l2s has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T21:21:48Z\n Ended: 2026-10-16T21:29:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4534656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q\", pod=\"ignition-server-proxy-d589845df-fjmtf\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q/ignition-server-proxy-d589845df-fjmtf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-16T21:21:48Z\n Ended: 2026-10-16T21:34:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4534656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q\", pod=\"network-node-identity-6bdbcc6c4c-jtfx4\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tg4162rq7ddnjp72dl91ufcjlaa7gdo-v1f7a0h3a7j8c7q/network-node-identity-6bdbcc6c4c-jtfx4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111173201227681792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7574,"pr_state":"closed","pr_sha":"a2b34108576b686569c2d8f8fc5e38cbda80d546","final_merged_sha":"fd0887d3395f226558878e178d6c4476bbb74aec","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T19:11:29Z","started_at":"2026-10-16T19:11:29Z","completed_at":"2026-10-16T21:43:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T21:58:56.402303088Z","timing_metadata_checked_at":"2026-10-16T21:58:56.402303088Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T21:58:56.406144576Z","region_metadata_checked_at":"2026-10-16T21:58:56.406144576Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111171489003737088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7587,"pr_state":"closed","pr_sha":"461c3676edb2549f9f0bbc26c7d16434e30e4463","final_merged_sha":"461c3676edb2549f9f0bbc26c7d16434e30e4463","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T19:04:41Z","started_at":"2026-10-16T19:04:41Z","completed_at":"2026-10-16T21:41:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T21:58:56.56841165Z","timing_metadata_checked_at":"2026-10-16T21:58:56.56841165Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T21:58:56.57130674Z","region_metadata_checked_at":"2026-10-16T21:58:56.57130674Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111149355237904384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7512,"pr_state":"closed","pr_sha":"925990167bded37212f2b08c9aab7672c09012ff","final_merged_sha":"925990167bded37212f2b08c9aab7672c09012ff","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T17:36:44Z","started_at":"2026-10-16T17:36:44Z","completed_at":"2026-10-16T20:00:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T20:13:57.670836429Z","timing_metadata_checked_at":"2026-10-16T20:13:57.670836429Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T20:13:57.674587313Z","region_metadata_checked_at":"2026-10-16T20:13:57.674587313Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111149108428279808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7590,"pr_state":"closed","pr_sha":"bf4d4e3b81f79d4befa13065846f702fea39524e","final_merged_sha":"bf4d4e3b81f79d4befa13065846f702fea39524e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T17:35:31Z","started_at":"2026-10-16T17:35:31Z","completed_at":"2026-10-16T19:59:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T20:13:57.802399973Z","timing_metadata_checked_at":"2026-10-16T20:13:57.802399973Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T20:13:57.80544696Z","region_metadata_checked_at":"2026-10-16T20:13:57.80544696Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111146056094846976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7580,"pr_state":"closed","pr_sha":"0c611949a02927c684b102343dcccc0663b92a88","final_merged_sha":"cc68e5f22f90cd600ec1702fd6051be82c4f0a33","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T17:23:37Z","started_at":"2026-10-16T17:23:37Z","completed_at":"2026-10-16T17:38:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T17:57:56.321413237Z","timing_metadata_checked_at":"2026-10-16T17:57:56.321413237Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T17:57:56.325421724Z","region_metadata_checked_at":"2026-10-16T17:57:56.325421724Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6e13b518450ca37364a56018ef3edc07b447e4a58dd945b77d911216e01b539e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111146056094846976","occurred_at":"2026-10-16T17:23:37Z","failed_at":"provision","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T17:28:31.305Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T17:28:32.589Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc","failure_pattern":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found","search_query":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111136882900340736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7580,"pr_state":"closed","pr_sha":"0c611949a02927c684b102343dcccc0663b92a88","final_merged_sha":"cc68e5f22f90cd600ec1702fd6051be82c4f0a33","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T16:47:10Z","started_at":"2026-10-16T16:47:10Z","completed_at":"2026-10-16T17:16:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T17:26:06.194407Z","timing_metadata_checked_at":"2026-10-16T17:26:06.194407Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T17:26:06.198861385Z","region_metadata_checked_at":"2026-10-16T17:26:06.198861385Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"04687ffe4c89a169795aec85dbe3fe3d6fade318e21adb4ea03c351c9809b08a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111136882900340736","occurred_at":"2026-10-16T16:47:10Z","failed_at":"provision","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T17:08:49.998Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T17:08:51.188Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc","failure_pattern":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found","search_query":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111133911231762432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7588,"pr_state":"open","pr_sha":"182655c91f5f3f82bdf9fc18dae52013845993e3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T16:35:11Z","started_at":"2026-10-16T16:35:11Z","completed_at":"2026-10-16T18:56:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T19:11:05.101753316Z","timing_metadata_checked_at":"2026-10-16T19:11:05.101753316Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T19:11:05.105187104Z","region_metadata_checked_at":"2026-10-16T19:11:05.105187104Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"97144b84fad012251125f4c881d9f837dc75cce7b7b2fd0d5fc36cb3df70bae8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111133911231762432","occurred_at":"2026-10-16T16:35:11Z","failed_at":"e2e","signature_id":"b689e896267fb659ab2567cecd3a930517666e0cf7750b5517f7c69837dd5c9d","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a1e5c39e140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lsp7k9q4q2pz\" to finish creating: GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\",\n \"name\": \"eeecd748-7cd9-428c-bdad-6172bba0e52f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T17:39:40.172421101Z\",\n \"endTime\": \"2026-10-16T17:58:48.49629868Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-pb4xk-8sfrz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-lsp7k9q4q2pz--managed/ingress-cert-le-np-1-pb4xk-8sfrz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-pb4xk-8sfrz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\n correlation ID: 3ee22f63-ca3a-4588-b9c5-ce97876471df\n client request ID: a644f8be-e878-4c8d-99e9-321877a22983\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a1e5c39e140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lsp7k9q4q2pz\" to finish creating: GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\",\n \"name\": \"eeecd748-7cd9-428c-bdad-6172bba0e52f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T17:39:40.172421101Z\",\n \"endTime\": \"2026-10-16T17:58:48.49629868Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-pb4xk-8sfrz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-pb4xk-8sfrz: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-lsp7k9q4q2pz--managed/ingress-cert-le-np-1-pb4xk-8sfrz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-lsp7k9q4q2pz--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-pb4xk-8sfrz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1762432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eeecd748-7cd9-428c-bdad-6172bba0e52f\n correlation ID: 3ee22f63-ca3a-4588-b9c5-ce97876471df\n client request ID: a644f8be-e878-4c8d-99e9-321877a22983","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lsp7k9q4q2pz\" to finish creating: GET https://rp"}},{"row_id":"a984dd0a1ecfc7bde34b8679f5dc0272ba392135fd63ad4ac435a59cae2ed591","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111133911231762432","occurred_at":"2026-10-16T16:35:11Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T17:47:40Z\n Ended: 2026-10-16T17:58:40Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1762432-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"c1219146cbadd70655f826f2a52e84163c5f3943216aaf0003b562cbbb9d6528","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111133911231762432","occurred_at":"2026-10-16T16:35:11Z","failed_at":"alert","signature_id":"3c51f31c3e12b890ebd28cd56bf13c145f5b755eda048613b9928abf2a5adabe","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T17:57:51Z\n Ended: 2026-10-16T18:04:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1762432-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tg1320l1nm4kt4ln02jlh4ms9qcvrv7-v20260901\", pod=\"openshift-apiserver-7f89dd8c4b-sck4g\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tg1320l1nm4kt4ln02jlh4ms9qcvrv7-v20260901/openshift-apiserver-7f89dd8c4b-sck4g has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111133275832455168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7305,"pr_state":"closed","pr_sha":"efd4ab7d28c76c9b2a798bf73755f8dbcfde1d21","final_merged_sha":"efd4ab7d28c76c9b2a798bf73755f8dbcfde1d21","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T16:32:50Z","started_at":"2026-10-16T16:32:50Z","completed_at":"2026-10-16T19:00:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T19:11:04.9216924Z","timing_metadata_checked_at":"2026-10-16T19:11:04.9216924Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T19:11:04.924272892Z","region_metadata_checked_at":"2026-10-16T19:11:04.924272892Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111132727754362880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"closed","pr_sha":"65be1906094a069aebc40f8ce2bbeeac23f40d2d","final_merged_sha":"8d693aca9ae999817853c4e772be53c43796d75a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T16:30:39Z","started_at":"2026-10-16T16:30:39Z","completed_at":"2026-10-16T18:52:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T19:11:04.643623403Z","timing_metadata_checked_at":"2026-10-16T19:11:04.643623403Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T19:11:04.647333391Z","region_metadata_checked_at":"2026-10-16T19:11:04.647333391Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111124389293461504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7556,"pr_state":"closed","pr_sha":"a77865bb4d11393400ce71e18662c65e0dec7cbe","final_merged_sha":"098b7e743ce49a5b70c3d964d3cdd247a796ee1d","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T15:57:31Z","started_at":"2026-10-16T15:57:31Z","completed_at":"2026-10-16T18:23:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T18:38:50.599373837Z","timing_metadata_checked_at":"2026-10-16T18:38:50.599373837Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T18:38:50.602664526Z","region_metadata_checked_at":"2026-10-16T18:38:50.602664526Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111124374575648768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7587,"pr_state":"closed","pr_sha":"461c3676edb2549f9f0bbc26c7d16434e30e4463","final_merged_sha":"461c3676edb2549f9f0bbc26c7d16434e30e4463","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T15:57:28Z","started_at":"2026-10-16T15:57:28Z","completed_at":"2026-10-16T18:14:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T18:28:50.240401268Z","timing_metadata_checked_at":"2026-10-16T18:28:50.240401268Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T18:28:50.243680057Z","region_metadata_checked_at":"2026-10-16T18:28:50.243680057Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"765cc041bf5a18cfdaccd4dc0e6471dedea3db9957cc1c95755dac00ab31a795","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111124374575648768","occurred_at":"2026-10-16T15:57:28Z","failed_at":"alert","signature_id":"dfb324349c1f93796329f7da2f6b37edd36593be9c02584b4eddb23c7aa504b8","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T17:00:31Z\n Ended: 2026-10-16T17:16:34Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5648768-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T17:00:31Z\n Ended: 2026-10-16T17:14:30Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5648768-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111121052871430144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6712,"pr_state":"open","pr_sha":"9bfd9800d00b7d8a9afc6860ec1c4a34d8307a77","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T15:44:16Z","started_at":"2026-10-16T15:44:16Z","completed_at":"2026-10-16T18:22:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T18:38:50.210081757Z","timing_metadata_checked_at":"2026-10-16T18:38:50.210081757Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T18:38:50.21554024Z","region_metadata_checked_at":"2026-10-16T18:38:50.21554024Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111120828451000320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"open","pr_sha":"9af097ada0a187a98f45aa51a8d8a1b962874a5c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T15:43:22Z","started_at":"2026-10-16T15:43:22Z","completed_at":"2026-10-16T18:23:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T18:38:50.342326143Z","timing_metadata_checked_at":"2026-10-16T18:38:50.342326143Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T18:38:50.352987609Z","region_metadata_checked_at":"2026-10-16T18:38:50.352987609Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"660334686f6d441f698719267140ceaa662114f762e250ed139930db5353f125","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111120828451000320","occurred_at":"2026-10-16T15:43:22Z","failed_at":"e2e","signature_id":"ab2ea88c29cdd316e2d7459a55135ddbe050949b4510bacb15d8ebb39608eed7","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3222464a4800\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\",\n err: \u003c*errors.errorString | 0x322245a3b8b0\u003e{\n s: \"dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3222464a4800\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-cdzxpnm9wgtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\",\n err: \u003c*errors.errorString | 0x322245a3b8b0\u003e{\n s: \"dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-2tg0gb0ik8kqufv9aj7d0iha95qjnf3j-e7g2r4n0l4l5r1h router=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000: expected matching summary after test start and within 8m; summaryUID=0fa463c2-2001-466d-92cc-dfa36a320846 node=aks-userswft3-10989426-vmss000000 pass=2b7ade69-c160-46d5-8524-479bdeb90f91 at=2026-10-16T17:09:43Z\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5529031a8c5e7aeec420f57c88f8725911bfa302f5c294049d97b239d627c655","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/westus3/ci01-j1000320-mgmt-1/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5529031a8c5e7aeec420f57c88f8725911bfa302f5c294049d97b239d627c655"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111119736791109632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7573,"pr_state":"open","pr_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T15:39:02Z","started_at":"2026-10-16T15:39:02Z","completed_at":"2026-10-16T18:13:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T18:28:49.841418663Z","timing_metadata_checked_at":"2026-10-16T18:28:49.841418663Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T18:28:49.847392644Z","region_metadata_checked_at":"2026-10-16T18:28:49.847392644Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"814d89e3919077b1a057a1ff46e716f9cc1219223871955b7f11cf268615137a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111119736791109632","occurred_at":"2026-10-16T15:39:02Z","failed_at":"alert","signature_id":"5d37298322872c4c9c58f6b51822a5ad1b91a1e040cee73204cb2fb09a29f82b","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T17:29:20Z\n Ended: 2026-10-16T17:35:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1109632-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tg0fmg5i9694ug5ff0i7ru6ntjv7r87-v2a4k0f7j4n8v4m\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tg0fmg5i9694ug5ff0i7ru6ntjv7r87-v2a4k0f7j4n8v4m/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7581/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111118479460405248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7581,"pr_state":"closed","pr_sha":"16aef998b08d964169ba51c62d406cacfa650bee","final_merged_sha":"16aef998b08d964169ba51c62d406cacfa650bee","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T15:34:02Z","started_at":"2026-10-16T15:34:02Z","completed_at":"2026-10-16T18:11:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T18:28:50.05175028Z","timing_metadata_checked_at":"2026-10-16T18:28:50.05175028Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T18:28:50.055182769Z","region_metadata_checked_at":"2026-10-16T18:28:50.055182769Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111118292637716480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7480,"pr_state":"closed","pr_sha":"b529e63aecbea8b9e7fed459fd90deb12074817d","final_merged_sha":"230a5d27c416299f628f0667ba08deeea65c4c88","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T15:33:18Z","started_at":"2026-10-16T15:33:18Z","completed_at":"2026-10-16T18:19:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T18:38:50.48020571Z","timing_metadata_checked_at":"2026-10-16T18:38:50.48020571Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T18:38:50.489118782Z","region_metadata_checked_at":"2026-10-16T18:38:50.489118782Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111110619204685824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"51196d4f7c3774ca92ab8dcab767e01796862041","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T15:02:46Z","started_at":"2026-10-16T15:02:46Z","completed_at":"2026-10-16T15:53:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T16:02:45.132632155Z","timing_metadata_checked_at":"2026-10-16T16:02:45.132632155Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T16:02:45.138308837Z","region_metadata_checked_at":"2026-10-16T16:02:45.138308837Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"dfb70c1c957b6b63cf5838ddbf326f6f1d2be6d8fe86485cc9a00605d61eb68c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111110619204685824","occurred_at":"2026-10-16T15:02:46Z","failed_at":"provision","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T15:46:44.545Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-16T15:46:45.068Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:d3bdf5672c862ebc0a35ea497266f8c7598862b9377e076ed14ffc329be19981]\"\ntime=2026-10-16T15:46:45.068Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-16T15:46:45.193Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-10-16T15:46:45.343Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-10-16T15:46:45.343Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-16T15:46:45.452Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-16T15:47:32.298Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-16T15:47:32.298Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111108963322826752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"924de57c9ad6f6049da7964e3f0bebdf0cf7eb7a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T14:56:13Z","started_at":"2026-10-16T14:56:13Z","completed_at":"2026-10-16T17:23:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T17:36:36.043814643Z","timing_metadata_checked_at":"2026-10-16T17:36:36.043814643Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T17:36:36.047617331Z","region_metadata_checked_at":"2026-10-16T17:36:36.047617331Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9c8e6360356981c9fca309073092287c24b251c948952f469bac8fa4a1c787b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111108963322826752","occurred_at":"2026-10-16T14:56:13Z","failed_at":"e2e","signature_id":"1a4ee126e99baefb3096a97ac2cd1348a9667da209b6831ca0327a37c66e723e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x225983c28ba0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x225983c28b60\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x225983a80280\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x225983c28b20\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x22598544d170\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x225983c28ba0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x225983c28b60\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x225983a80280\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x225983c28b20\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-69hzvgr8h68z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T15:59:20.1954973Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x22598544d170\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111108036918513664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7483,"pr_state":"open","pr_sha":"89d51010763b0541b5dee945f23dbea714b56ec5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T14:52:33Z","started_at":"2026-10-16T14:52:33Z","completed_at":"2026-10-16T17:23:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T17:36:36.356580749Z","timing_metadata_checked_at":"2026-10-16T17:36:36.356580749Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T17:36:36.359968138Z","region_metadata_checked_at":"2026-10-16T17:36:36.359968138Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7512,"pr_state":"closed","pr_sha":"925990167bded37212f2b08c9aab7672c09012ff","final_merged_sha":"925990167bded37212f2b08c9aab7672c09012ff","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T14:51:17Z","started_at":"2026-10-16T14:51:17Z","completed_at":"2026-10-16T16:24:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T16:34:15.030923342Z","timing_metadata_checked_at":"2026-10-16T16:34:15.030923342Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T16:34:15.036783327Z","region_metadata_checked_at":"2026-10-16T16:34:15.036783327Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"2d775de146ca468f756e8ff5d86bf93e70800314ba6183d61393334acff021e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","occurred_at":"2026-10-16T14:51:17Z","failed_at":"alert","signature_id":"114cc78e60d3d816b196ff9d1e4915479d09d11c74f7d7fb5230273971e76a73","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-16T16:17:41Z\n Severity: Sev2\n Labels: alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j7660416-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.75:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7660416/providers/microsoft.monitor/accounts/services-j7660416\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-llr7w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-16T16:17:41Z\n Severity: Sev2\n Labels: alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j7660416-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.75:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7660416/providers/microsoft.monitor/accounts/services-j7660416\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-llr7w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e","failure_pattern":"alert [svc] MaestroServerNoReadyReplicas fired","search_query":"alert [svc] MaestroServerNoReadyReplicas fired"}},{"row_id":"a82b256dd33b4823ff5b80e596c8da76957f072cb5448ac9b188849525310a77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","occurred_at":"2026-10-16T14:51:17Z","failed_at":"alert","signature_id":"d07de28f589089258ca9dc6edac09be43e6962d6a3ef23500a1585d2a82b650c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 18 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-16T16:03:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-c4d6b786-548pt\", region=\"westus3\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-c4d6b786-548pt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-5lt2p\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-5lt2p has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-nw2d2\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-nw2d2 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-rsnz7\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-rsnz7 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-q6lgt\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-q6lgt has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-27216613-vmss000000-jg5jx\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-27216613-vmss000000-jg5jx has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"hubble-relay-58594b5dd7-c85j5\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/hubble-relay-58594b5dd7-c85j5 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-9j44l\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-9j44l has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-27216613-vmss000002-wgh5j\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-27216613-vmss000002-wgh5j has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-27216613-vmss000001-zvc5c\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-27216613-vmss000001-zvc5c has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-16T16:05:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-bjvb8\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-bjvb8 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-16T16:13:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-6cb67d7f44-f4tbd\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-6cb67d7f44-f4tbd has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-16T16:13:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-77fbb4f5f5-2fjpq\", region=\"westus3\", severity=\"warning\"\n Description: Pod sessiongate/sessiongate-77fbb4f5f5-2fjpq has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-16T16:13:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-6cb67d7f44-89fpd\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-6cb67d7f44-89fpd has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-16T16:13:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-6cb67d7f44-g6qp6\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-6cb67d7f44-g6qp6 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-16T16:13:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-7dfb5b7d6b-dgw7q\", region=\"westus3\", severity=\"warning\"\n Description: Pod maestro/maestro-7dfb5b7d6b-dgw7q has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-16T16:13:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-7dfb5b7d6b-qws2j\", region=\"westus3\", severity=\"warning\"\n Description: Pod maestro/maestro-7dfb5b7d6b-qws2j has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-16T16:13:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7660416-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-77fbb4f5f5-gdqfm\", region=\"westus3\", severity=\"warning\"\n Description: Pod sessiongate/sessiongate-77fbb4f5f5-gdqfm has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"d74aa8a98d0d4aa50d6760643713b5fa0e4ac3dd96defa2747b2d5ab7b0dce3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","occurred_at":"2026-10-16T14:51:17Z","failed_at":"alert","signature_id":"26ad01559a977d2ccbf9de49f0e0a12ad85e844d43c5c076830f46ba6268215f","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T15:54:14Z\n Ended: 2026-10-16T16:02:12Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j7660416-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T15:55:11Z\n Ended: 2026-10-16T16:02:12Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j7660416-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}},{"row_id":"f403fe60b1308cd5ea3decaf773a87b308e02477e18f7a53dde2566708d2c267","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7512/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111107720617660416","occurred_at":"2026-10-16T14:51:17Z","failed_at":"provision","signature_id":"f7a2f83e4294c3ba837340ee15ecf8b39ec31b12220850df230141abca88bbe0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T16:06:45.672Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-16T16:06:46.344Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values=\"map[azureArmHelperIdentityCertName:armHelperPoolCert-38 azureArmHelperIdentityClientId:6608bb27-dfa1-4c02-8395-4ca010bd73db azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:d0cc3060-1e84-4147-a32a-544fe7e53387 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-16 azureMiMockServicePrincipalClientId:33e65307-a696-491e-831f-42341ce3a9b5 azureMiMockServicePrincipalPrincipalId:b26c2343-0863-4a55-bb7e-dfb2544999c1 azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j7660416 cosmosDatabaseUrl:https://arohcpci01-rp-j7660416.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:e9d0c188179de6a6eb745bf8b91e2a9cf54424a74b55ce29ef87ab06cf658ddb] imageDigest:sha256:e9d0c188179de6a6eb745bf8b91e2a9cf54424a74b55ce29ef87ab06cf658ddb imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj7660416.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj7660416.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:0531f05d-8afc-4435-86c7-b9a82d26a229 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:westus3 replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]\"\ntime=2026-10-16T16:06:46.344Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-16T16:06:46.586Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service\ntime=2026-10-16T16:06:46.981Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false\ntime=2026-10-16T16:06:46.981Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-16T16:06:47.117Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-16T16:16:56.303Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-10-16T16:16:56.304Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a8bb4542e08df91dbf01364e592eda2b949d63ea36dbf43a83cbdbd4c51b7dfe","failure_pattern":"failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","a8bb4542e08df91dbf01364e592eda2b949d63ea36dbf43a83cbdbd4c51b7dfe","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111097106860085248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7132,"pr_state":"closed","pr_sha":"823aadd55f050b7e4bb63d67795c10122e0d976f","final_merged_sha":"9f60977a89121124186b3a9255a54b3e5c0b283b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T14:09:07Z","started_at":"2026-10-16T14:09:07Z","completed_at":"2026-10-16T16:26:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T16:44:49.909622868Z","timing_metadata_checked_at":"2026-10-16T16:44:49.909622868Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T16:44:49.913175954Z","region_metadata_checked_at":"2026-10-16T16:44:49.913175954Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111095141568614400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7584,"pr_state":"closed","pr_sha":"c47eebcf4d76a88b9a391565c866d5dfcfa09563","final_merged_sha":"c47eebcf4d76a88b9a391565c866d5dfcfa09563","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T14:01:06Z","started_at":"2026-10-16T14:01:06Z","completed_at":"2026-10-16T16:15:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T16:34:15.230678644Z","timing_metadata_checked_at":"2026-10-16T16:34:15.230678644Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T16:34:15.234131036Z","region_metadata_checked_at":"2026-10-16T16:34:15.234131036Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111090334573268992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"closed","pr_sha":"ae400b1a0b760e7afcb6471a6e4453c50bf4febe","final_merged_sha":"7f136ea439f0e69b9cacf1a3a044164c9c5ac85f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T13:42:12Z","started_at":"2026-10-16T13:42:12Z","completed_at":"2026-10-16T15:53:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T16:02:44.979543955Z","timing_metadata_checked_at":"2026-10-16T16:02:44.979543955Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T16:02:44.98389094Z","region_metadata_checked_at":"2026-10-16T16:02:44.98389094Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6bdfd8d421c5f277b8d831ad35b744712a207d29b429fa0c618cd0b205a2c0d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111090334573268992","occurred_at":"2026-10-16T13:42:12Z","failed_at":"e2e","signature_id":"c6a5ed957cb1161fe411a2fb10ae2652cf0af6041a491232f29a014245c182c2","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a4a4982c560\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2a4a4982c540\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x2a4a477c0190\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x2a4a4982c520\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x2a4a49889830\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2a4a4982c560\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2a4a4982c540\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x2a4a477c0190\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x2a4a4982c520\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-97wmvqc47hmc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:33:12.4130712Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x2a4a49889830\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}},{"row_id":"81e076ac003a2abef12cbf8a4465a3089b81fffab0fad4a065563b7e8766bfcf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111090334573268992","occurred_at":"2026-10-16T13:42:12Z","failed_at":"alert","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T14:54:18Z\n Ended: 2026-10-16T15:01:19Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3268992-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"955243522c20bfbc7c5d8f5c6ee85b99c9b960bc3f2230f430b6b6b92f143508","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111090334573268992","occurred_at":"2026-10-16T13:42:12Z","failed_at":"e2e","signature_id":"013f773a3c97f51390c5446d7bcbcb1250a51911bae762a125e99650623c4fcd","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:467]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23efaa396df0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:467]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x23efaa396df0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7572/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111089245203468288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7572,"pr_state":"closed","pr_sha":"d975aa3c93d9125fca961c917744e1b9aff813ea","final_merged_sha":"d975aa3c93d9125fca961c917744e1b9aff813ea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T13:37:52Z","started_at":"2026-10-16T13:37:52Z","completed_at":"2026-10-16T15:53:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T16:02:45.334892795Z","timing_metadata_checked_at":"2026-10-16T16:02:45.334892795Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T16:02:45.338536283Z","region_metadata_checked_at":"2026-10-16T16:02:45.338536283Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111088872313065472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7577,"pr_state":"closed","pr_sha":"161fd8022ee5bfd694e89f51a023a4671cc30525","final_merged_sha":"161fd8022ee5bfd694e89f51a023a4671cc30525","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T13:36:23Z","started_at":"2026-10-16T13:36:23Z","completed_at":"2026-10-16T15:49:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T16:02:45.493559978Z","timing_metadata_checked_at":"2026-10-16T16:02:45.493559978Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T16:02:45.497446165Z","region_metadata_checked_at":"2026-10-16T16:02:45.497446165Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c09679488471a8b46eb49c77a27dc4981689c665bbaa77b7a9e884e0424df7bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111088872313065472","occurred_at":"2026-10-16T13:36:23Z","failed_at":"e2e","signature_id":"fb1ea24c5e9ac0e2100d89dced160bc12a43efdc23d9fc0e7e7e9dd8e55ddc2a","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x281ad7cc620\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x281ad7cc600\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x281ad410320\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x281ad7cc5e0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x281ae968cc0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x281ad7cc620\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x281ad7cc600\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x281ad410320\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x281ad7cc5e0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-sh9nsvf7xg7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:22:29.6817536Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x281ae968cc0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111087082628714496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"924de57c9ad6f6049da7964e3f0bebdf0cf7eb7a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T13:29:17Z","started_at":"2026-10-16T13:29:17Z","completed_at":"2026-10-16T14:41:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T15:00:12.039334807Z","timing_metadata_checked_at":"2026-10-16T15:00:12.039334807Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T15:00:12.042644497Z","region_metadata_checked_at":"2026-10-16T15:00:12.042644497Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"5c6cc73e8f010ba555d5e87d767975945d5ef6401094595f68009ce280f77bde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111087082628714496","occurred_at":"2026-10-16T13:29:17Z","failed_at":"alert","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-16T14:26:06Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j8714496-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}},{"row_id":"70dcf10b501e97f26da254a6fe61b0c693bfc011382a8a2f286e5dd478a3bc66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111087082628714496","occurred_at":"2026-10-16T13:29:17Z","failed_at":"provision","signature_id":"2a6bf4d1ff2fafa7ef97169789e44726bf300fcadbf9dec849e1efa942bd4a0d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T14:04:27.353Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-16T14:34:26.879Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-16T14:04:27.363830736Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-16T14:04:29.161200885Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-16T14:04:29.579255531Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75ba2f620eb9a6826cee085233900a138186175271fd9f54c13e5c69036cc8d1","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j8714496-mgmt-1\",\"clusterName\":\"ci01-j871449","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-16T14:04:27.363830736Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-16T14:04:29.161200885Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-16T14:04:29.579255531Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j8714496-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j8714496-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["75ba2f620eb9a6826cee085233900a138186175271fd9f54c13e5c69036cc8d1","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111084766907338752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7582,"pr_state":"closed","pr_sha":"54bb3b8e043163b9f9ea058dace1451f9385c3f5","final_merged_sha":"a1dd5b74b0d0373c6ecafbf826ea37646763f7a7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T13:20:05Z","started_at":"2026-10-16T13:20:05Z","completed_at":"2026-10-16T15:37:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T15:52:08.200866788Z","timing_metadata_checked_at":"2026-10-16T15:52:08.200866788Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T15:52:08.204691176Z","region_metadata_checked_at":"2026-10-16T15:52:08.204691176Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2a9731240585c1b4a766d875c5ee53e855b95da95ae769106546c89ae0424127","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111084766907338752","occurred_at":"2026-10-16T13:20:05Z","failed_at":"e2e","signature_id":"f39c90cd5040db84704fbb6ee3d67b497be28152e614a615d759d414b915f724","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x102cc34803e0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x102cc34803c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x102cc29781e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x102cc34803a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x102cc44c70b0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x102cc34803e0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x102cc34803c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x102cc29781e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x102cc34803a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v6ftbgcswsq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T14:12:28.9453936Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x102cc44c70b0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}},{"row_id":"cd42a74f62769cae94ed351042611bf2e5d9cff9c4fb7c69af8e832e9885d590","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111084766907338752","occurred_at":"2026-10-16T13:20:05Z","failed_at":"e2e","signature_id":"af2923d29e8301ef3cd00e5f05932057532b4d49d8d8bcf7e33cfa90a1f4a5a8","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x340dc62fa018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dsg52jhcqb45\" to finish creating: GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\",\n \"name\": \"baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T14:26:03.913560661Z\",\n \"endTime\": \"2026-10-16T14:45:08.626305681Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\n correlation ID: 85e240f2-d0e4-4c29-bd30-d4d4327881c2\n client request ID: d5c7b389-b2c6-4aba-9dfa-9055479c941f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x340dc62fa018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dsg52jhcqb45\" to finish creating: GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\",\n \"name\": \"baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T14:26:03.913560661Z\",\n \"endTime\": \"2026-10-16T14:45:08.626305681Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7338752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baecf025-dfba-4e06-b1e1-28ab3b4cbfe6\n correlation ID: 85e240f2-d0e4-4c29-bd30-d4d4327881c2\n client request ID: d5c7b389-b2c6-4aba-9dfa-9055479c941f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111079254782382080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7556,"pr_state":"closed","pr_sha":"47e22d679b44a818603a0e31f72c39d9b0de2846","final_merged_sha":"098b7e743ce49a5b70c3d964d3cdd247a796ee1d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T12:58:10Z","started_at":"2026-10-16T12:58:10Z","completed_at":"2026-10-16T15:03:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T15:21:09.319005523Z","timing_metadata_checked_at":"2026-10-16T15:21:09.319005523Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T15:21:09.322163514Z","region_metadata_checked_at":"2026-10-16T15:21:09.322163514Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"31cdc6836e91b281379633841f444d94d155082821ea150d7b792a25ece5a5ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111079254782382080","occurred_at":"2026-10-16T12:58:10Z","failed_at":"e2e","signature_id":"f1597e364c1f86f7d3afaa02ce110c4da89f5136194711d5b0ee2274d8ec6008","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x2dbe92ada858\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-t6jslt7lwjkt\" to finish creating: GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\",\n \"name\": \"88e58151-a475-4725-8e82-2d51b0f7fb69\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T13:50:29.440992073Z\",\n \"endTime\": \"2026-10-16T14:09:32.07987309Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-5p4h6-brpt4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-5p4h6-brpt4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-5p4h6-brpt4: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-t6jslt7lwjkt--managed/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/availabilitySets/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '10' seconds after a... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\n correlation ID: 054f0541-4905-4acf-9baf-353f2e747fdd\n client request ID: 320b7448-1575-4fdb-9611-9bf748713191\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x2dbe92ada858\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-t6jslt7lwjkt\" to finish creating: GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\",\n \"name\": \"88e58151-a475-4725-8e82-2d51b0f7fb69\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T13:50:29.440992073Z\",\n \"endTime\": \"2026-10-16T14:09:32.07987309Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-5p4h6-brpt4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-5p4h6-brpt4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-5p4h6-brpt4: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-t6jslt7lwjkt--managed/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-t6jslt7lwjkt--managed/providers/Microsoft.Compute/availabilitySets/2tftmlmmdno0igptnhfrk6md9uivpr81_cilium-cl-cilium-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '10' seconds after a... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2382080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/88e58151-a475-4725-8e82-2d51b0f7fb69\n correlation ID: 054f0541-4905-4acf-9baf-353f2e747fdd\n client request ID: 320b7448-1575-4fdb-9611-9bf748713191\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6494a57235b1e5ea0036c9f08ed717c4d99f3e06862fbe4cb98568bb3fb00c61","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after a... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c3535b389d475b7e1a94df4a0834ca42edbdad4ed7c7902cd09dd560ed99043d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111079254782382080","occurred_at":"2026-10-16T12:58:10Z","failed_at":"e2e","signature_id":"b2293d4fa65dfff28dd996462bab11c3af45d26e4fa09b72ed06e5be8af20cbf","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x19d0afcab040\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x19d0afcab020\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x19d0af322be0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x19d0afcab000\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x19d0b1181830\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x19d0afcab040\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x19d0afcab020\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x19d0af322be0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x19d0afcab000\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-nc456qjx64bv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:39:56.2350122Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x19d0b1181830\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6494a57235b1e5ea0036c9f08ed717c4d99f3e06862fbe4cb98568bb3fb00c61","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111076959755702272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7539,"pr_state":"closed","pr_sha":"6097a92abd96718069c7597a4f512c0e358a9b04","final_merged_sha":"6097a92abd96718069c7597a4f512c0e358a9b04","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T12:49:03Z","started_at":"2026-10-16T12:49:03Z","completed_at":"2026-10-16T14:57:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T15:10:58.448994183Z","timing_metadata_checked_at":"2026-10-16T15:10:58.448994183Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T15:10:58.452441072Z","region_metadata_checked_at":"2026-10-16T15:10:58.452441072Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1cf395ad937c7806a72df8de784ce358c1d3dea6f27d3723e1e6bc8658211063","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7539/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111076959755702272","occurred_at":"2026-10-16T12:49:03Z","failed_at":"e2e","signature_id":"e62ea6a200fd389ea519e04ea59463bd4151415eaccd997a4c214c2e3763a28a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x26aec5b792f0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:365]: Timed out after 300.003s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x26aec5b792f0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40106-\u003e172.30.0.10:53: read: connection refused\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111074925870583808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7292,"pr_state":"closed","pr_sha":"e2249e5ba4dbcc59216d7341da5783394710eb68","final_merged_sha":"e2249e5ba4dbcc59216d7341da5783394710eb68","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T12:40:38Z","started_at":"2026-10-16T12:40:38Z","completed_at":"2026-10-16T14:55:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T15:10:58.233009185Z","timing_metadata_checked_at":"2026-10-16T15:10:58.233009185Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T15:10:58.236510974Z","region_metadata_checked_at":"2026-10-16T15:10:58.236510974Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111072619372482560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"closed","pr_sha":"09587f6f09c91b53ea705711a675d523def6cece","final_merged_sha":"09587f6f09c91b53ea705711a675d523def6cece","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T12:31:48Z","started_at":"2026-10-16T12:31:48Z","completed_at":"2026-10-16T14:41:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T15:00:12.256502399Z","timing_metadata_checked_at":"2026-10-16T15:00:12.256502399Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T15:00:12.259380389Z","region_metadata_checked_at":"2026-10-16T15:00:12.259380389Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7523/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111072522505031680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7523,"pr_state":"open","pr_sha":"17238efd860b5907bd80b8dd8dda0bd781379a5a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T12:31:25Z","started_at":"2026-10-16T12:31:25Z","completed_at":"2026-10-16T14:36:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T14:50:10.455675555Z","timing_metadata_checked_at":"2026-10-16T14:50:10.455675555Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T14:50:10.459716042Z","region_metadata_checked_at":"2026-10-16T14:50:10.459716042Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4cbd6added829493694faa24ba69560dc3ccf6f467f67eb5f068595c841771f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7523/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111072522505031680","occurred_at":"2026-10-16T12:31:25Z","failed_at":"e2e","signature_id":"78562743ea1326baf0fb7954282c6ad239a40798722c5ed5c25f650423266f08","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n \u003c*exported.ResponseError | 0x31fef30d4a50\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n correlation ID: 63080195-ea99-42b1-8144-1ce2de77a2d8\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n \u003c*exported.ResponseError | 0x31fef30d4a50\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-04-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n correlation ID: 63080195-ea99-42b1-8144-1ce2de77a2d8\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429"}},{"row_id":"f19336ca0aac7d50e79ac818b711e50ecec8f8821c222af381fc9153a8f06fb8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7523/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111072522505031680","occurred_at":"2026-10-16T12:31:25Z","failed_at":"e2e","signature_id":"78897419cb3b43b7451376611892509b6c60c0d6e400e895529a13bc21831597","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x33e95d68a9c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x33e95d68a9a0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x33e95c4f2d20\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x33e95d68a980\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x33e95d14a480\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x33e95d68a9c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x33e95d68a9a0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x33e95c4f2d20\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x33e95d68a980\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qds2jlzpklhh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T13:12:53.4717042Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x33e95d14a480\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111067237912678400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7489,"pr_state":"closed","pr_sha":"264e26cad81219a68bc04354e7ba4db0ff65845a","final_merged_sha":"a800194beb9b2fac67229770cb4864563f1e4d17","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T12:10:25Z","started_at":"2026-10-16T12:10:25Z","completed_at":"2026-10-16T14:29:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T14:39:50.041160393Z","timing_metadata_checked_at":"2026-10-16T14:39:50.041160393Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T14:39:50.043970984Z","region_metadata_checked_at":"2026-10-16T14:39:50.043970984Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7f02aaa55cb6ec0d95818c2f183e89ae1505b76d6f9d053af187eaf0205d60bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111067237912678400","occurred_at":"2026-10-16T12:10:25Z","failed_at":"e2e","signature_id":"504867c83cb57dcb50cb4595712ff6bc2c40f73af164bfc8cf71f5840b752867","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x239a48fa84c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x239a4a108640\u003e{\n s: \"dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x239a48fa84c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jv8jpqmq9gpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x239a4a108640\u003e{\n s: \"dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-2tft4ek4kogcffngbs1nats12glqcn7n-x6a3m7l5u9v6a9d: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91847fc55e97ea83d42ed15b0d83f685c1e0a138fec204f0e908e2bf134a6478","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j2678400-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["91847fc55e97ea83d42ed15b0d83f685c1e0a138fec204f0e908e2bf134a6478"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111067129682857984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7382,"pr_state":"closed","pr_sha":"a5a34b4206704d35ef0c9bc32307a0c35b44d0a8","final_merged_sha":"a5a34b4206704d35ef0c9bc32307a0c35b44d0a8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T12:10:00Z","started_at":"2026-10-16T12:10:00Z","completed_at":"2026-10-16T14:27:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T14:39:49.838131447Z","timing_metadata_checked_at":"2026-10-16T14:39:49.838131447Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T14:39:49.842003434Z","region_metadata_checked_at":"2026-10-16T14:39:49.842003434Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7579/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111066626743865344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7579,"pr_state":"closed","pr_sha":"426fe4d7d8cbc4a1ba5f222abbdb10676470eb52","final_merged_sha":"426fe4d7d8cbc4a1ba5f222abbdb10676470eb52","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T12:08:00Z","started_at":"2026-10-16T12:08:00Z","completed_at":"2026-10-16T14:25:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T14:39:50.209454352Z","timing_metadata_checked_at":"2026-10-16T14:39:50.209454352Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T14:39:50.217147327Z","region_metadata_checked_at":"2026-10-16T14:39:50.217147327Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111057987362623488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7578,"pr_state":"open","pr_sha":"6df425210d858e28542f7860f553b8621228e9ca","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T11:33:40Z","started_at":"2026-10-16T11:33:40Z","completed_at":"2026-10-16T13:44:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T13:57:39.091336932Z","timing_metadata_checked_at":"2026-10-16T13:57:39.091336932Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T13:57:39.094625022Z","region_metadata_checked_at":"2026-10-16T13:57:39.094625022Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111046289121611776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7577,"pr_state":"closed","pr_sha":"9c60bcb76e1e2fb9163d82fc23381a72f5a6f957","final_merged_sha":"161fd8022ee5bfd694e89f51a023a4671cc30525","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T10:47:11Z","started_at":"2026-10-16T10:47:11Z","completed_at":"2026-10-16T12:56:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T13:15:59.533153074Z","timing_metadata_checked_at":"2026-10-16T13:15:59.533153074Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T13:15:59.536906361Z","region_metadata_checked_at":"2026-10-16T13:15:59.536906361Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6982,"pr_state":"open","pr_sha":"72835ce259b1910bd6815bd44e79ae4cf1460ac9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T10:29:17Z","started_at":"2026-10-16T10:29:17Z","completed_at":"2026-10-16T13:25:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T13:37:20.691974844Z","timing_metadata_checked_at":"2026-10-16T13:37:20.691974844Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T13:37:20.695360831Z","region_metadata_checked_at":"2026-10-16T13:37:20.695360831Z"},"failed_at":["alert","e2e"],"failed_test_count":72,"occurrences":[{"row_id":"00351ddfc2c458c7fa25ff7f6612bf604388014b33f3719544f99acd00a80490","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"089aef486c6063d842b930743dd8e473f8a41298562aa03e74f23d7f080684bb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-r564c69p7xg4/cluster-candidate-5-1-bphh8v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x25f3994d0160\u003e: \n failed to create HCP cluster cluster-candidate-5-1-bphh8v: failed waiting for cluster=\"cluster-candidate-5-1-bphh8v\" in resourcegroup=\"rg-candidate-5-1-r564c69p7xg4\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf\",\n \"name\": \"b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:46.450911767Z\",\n \"endTime\": \"2026-10-16T11:47:49.840205163Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf\n correlation ID: 9721cca3-00ee-40da-a65c-4dbeafe27604\n client request ID: 5c89be69-937f-4b9c-b3fe-2ac7fb2f335b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-r564c69p7xg4/cluster-candidate-5-1-bphh8v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x25f3994d0160\u003e: \n failed to create HCP cluster cluster-candidate-5-1-bphh8v: failed waiting for cluster=\"cluster-candidate-5-1-bphh8v\" in resourcegroup=\"rg-candidate-5-1-r564c69p7xg4\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf\",\n \"name\": \"b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:46.450911767Z\",\n \"endTime\": \"2026-10-16T11:47:49.840205163Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b86a5d73-90cf-4e1b-9bf8-cd8e1fb92bdf\n correlation ID: 9721cca3-00ee-40da-a65c-4dbeafe27604\n client request ID: 5c89be69-937f-4b9c-b3fe-2ac7fb2f335b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"014c4f453a3bd5d17ac84d505d5c414e2cb6b02661dcfb23e3f4c3af094b5364","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"7942f5dbb02de53dfe9b25dd229b9629fc800c349fe2c7f397013613de82a0a9","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x267b45e33940\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-wg6vfj68ppbj\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3\",\n \"name\": \"7582db6b-1ec3-41f5-ae34-034a5f0015a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.316842517Z\",\n \"endTime\": \"2026-10-16T11:47:34.729108911Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3\n correlation ID: 672d62e4-3264-40b7-a323-c9f0c7db8552\n client request ID: 45b671db-f5be-402f-845b-ad031d68dc35\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x267b45e33940\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-wg6vfj68ppbj\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3\",\n \"name\": \"7582db6b-1ec3-41f5-ae34-034a5f0015a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.316842517Z\",\n \"endTime\": \"2026-10-16T11:47:34.729108911Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7582db6b-1ec3-41f5-ae34-034a5f0015a3\n correlation ID: 672d62e4-3264-40b7-a323-c9f0c7db8552\n client request ID: 45b671db-f5be-402f-845b-ad031d68dc35\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"051bf63a46f9b839f1557a4e9e732e1c84d613fdd0cc7c1a8d3231070f94ff8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"7d04c13ed582cb2d55761c556286e5033b490f884607524524f1777b4ec41755","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-zqjt7w\" with version 4.22.17 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xef740c20200\u003e: \n failed to create HCP cluster cluster-zstream-4-22-zqjt7w: failed waiting for cluster=\"cluster-zstream-4-22-zqjt7w\" in resourcegroup=\"rg-zstream-upgrade-4-22-tgrmbldgf24k\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852\",\n \"name\": \"26a69e16-4222-4373-86da-5d09a67bc852\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:45.142598291Z\",\n \"endTime\": \"2026-10-16T11:47:49.837183985Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852\n correlation ID: 7f2b09a0-342f-4567-b3e6-0d9f66f0004d\n client request ID: 1f5c976b-37b6-4713-a33e-c70d3f34ed24\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-zqjt7w\" with version 4.22.17 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xef740c20200\u003e: \n failed to create HCP cluster cluster-zstream-4-22-zqjt7w: failed waiting for cluster=\"cluster-zstream-4-22-zqjt7w\" in resourcegroup=\"rg-zstream-upgrade-4-22-tgrmbldgf24k\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852\",\n \"name\": \"26a69e16-4222-4373-86da-5d09a67bc852\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:45.142598291Z\",\n \"endTime\": \"2026-10-16T11:47:49.837183985Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-21/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26a69e16-4222-4373-86da-5d09a67bc852\n correlation ID: 7f2b09a0-342f-4567-b3e6-0d9f66f0004d\n client request ID: 1f5c976b-37b6-4713-a33e-c70d3f34ed24\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"09df5a555f35c098d5c4f12a1f6c4f7c4e3dd7933253123a06b9070146dabbe4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"1d37cf0777cda01567c7d0c86c154a99941b6bfaea816e2afa28caec0e946b5a","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x302ea639cb20\u003e: \n failed to create HCP cluster np-dg-minor-84pjd6: failed waiting for cluster=\"np-dg-minor-84pjd6\" in resourcegroup=\"rg-np-dg-minor-84pjd6-pxcbzv9wv7lk\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a\",\n \"name\": \"28010958-6488-4e47-a97f-3c980e40e80a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:29.456420902Z\",\n \"endTime\": \"2026-10-16T12:43:31.119999655Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a\n correlation ID: a869227a-6203-4efd-a1fa-22d66ad064bb\n client request ID: dd4587bf-6acc-4c74-9458-b60c41fea5b3\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x302ea639cb20\u003e: \n failed to create HCP cluster np-dg-minor-84pjd6: failed waiting for cluster=\"np-dg-minor-84pjd6\" in resourcegroup=\"rg-np-dg-minor-84pjd6-pxcbzv9wv7lk\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a\",\n \"name\": \"28010958-6488-4e47-a97f-3c980e40e80a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:29.456420902Z\",\n \"endTime\": \"2026-10-16T12:43:31.119999655Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28010958-6488-4e47-a97f-3c980e40e80a\n correlation ID: a869227a-6203-4efd-a1fa-22d66ad064bb\n client request ID: dd4587bf-6acc-4c74-9458-b60c41fea5b3\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0a75596cf5d88763a341da7c527c606f4f0d98abab78c5d499abd86396b7ef8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"42e98ecdecbc4839cf618f859d9c4e963fd3f5fa82e5c8c1be6af5035a269d5c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-qxqtw472st5s/cluster-candidate-4-22-kpqx7c should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ec0d0beadc0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-kpqx7c: failed waiting for cluster=\"cluster-candidate-4-22-kpqx7c\" in resourcegroup=\"rg-candidate-4-22-qxqtw472st5s\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f\",\n \"name\": \"37bf4b59-12e7-4663-92b5-2fa49b9e914f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.404569434Z\",\n \"endTime\": \"2026-10-16T11:47:34.737960425Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f\n correlation ID: 26fc1ea5-6812-46a0-ab6d-df6b6366a988\n client request ID: 4115d0b3-eb51-4b91-b3f2-2e25d96cfb16\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-qxqtw472st5s/cluster-candidate-4-22-kpqx7c should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ec0d0beadc0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-kpqx7c: failed waiting for cluster=\"cluster-candidate-4-22-kpqx7c\" in resourcegroup=\"rg-candidate-4-22-qxqtw472st5s\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f\",\n \"name\": \"37bf4b59-12e7-4663-92b5-2fa49b9e914f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.404569434Z\",\n \"endTime\": \"2026-10-16T11:47:34.737960425Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/37bf4b59-12e7-4663-92b5-2fa49b9e914f\n correlation ID: 26fc1ea5-6812-46a0-ab6d-df6b6366a988\n client request ID: 4115d0b3-eb51-4b91-b3f2-2e25d96cfb16\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0baa632721b9dfd9a6735b60cd0aada04de550047959956b3a04a22c1e581eac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"20cab90c333e74780b28476d91e59f4c5e920fd33e4db9625862d05048d14bb9","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a9b1c8b6ba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-csxwdq5kg9sx\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6\",\n \"name\": \"1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:40.61907571Z\",\n \"endTime\": \"2026-10-16T11:47:49.830293866Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6\n correlation ID: a282e9f7-9468-4fa7-aefb-7fc6d52c0c49\n client request ID: 0fa2af39-92d5-4643-befd-c770b11ab6af\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a9b1c8b6ba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-csxwdq5kg9sx\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6\",\n \"name\": \"1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:40.61907571Z\",\n \"endTime\": \"2026-10-16T11:47:49.830293866Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1ce3de9a-ba26-4aac-b7ca-d6d0ab5fc8a6\n correlation ID: a282e9f7-9468-4fa7-aefb-7fc6d52c0c49\n client request ID: 0fa2af39-92d5-4643-befd-c770b11ab6af\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"111e573509254ede7a3e95826de638f0f781e3dc3d5e17ec7a0590bfc16fb660","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"701100ab256952d9a5de8b351e6a3b385a113afbfeffcd255acd7e6f24e3a347","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-rcc6r4\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x37d362890860\u003e: \n failed to create HCP cluster cluster-zstream-4-23-rcc6r4: failed waiting for cluster=\"cluster-zstream-4-23-rcc6r4\" in resourcegroup=\"rg-zstream-upgrade-4-23-q79v76gpnl59\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6\",\n \"name\": \"ab63a679-4670-419c-aaa4-e0251814e5e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.848366298Z\",\n \"endTime\": \"2026-10-16T11:47:34.735398098Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6\n correlation ID: 7e0a0830-9f92-4675-99de-dfd56a431231\n client request ID: 391dd89d-6617-4212-aab7-a94d3c4c617f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-rcc6r4\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x37d362890860\u003e: \n failed to create HCP cluster cluster-zstream-4-23-rcc6r4: failed waiting for cluster=\"cluster-zstream-4-23-rcc6r4\" in resourcegroup=\"rg-zstream-upgrade-4-23-q79v76gpnl59\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6\",\n \"name\": \"ab63a679-4670-419c-aaa4-e0251814e5e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.848366298Z\",\n \"endTime\": \"2026-10-16T11:47:34.735398098Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ab63a679-4670-419c-aaa4-e0251814e5e6\n correlation ID: 7e0a0830-9f92-4675-99de-dfd56a431231\n client request ID: 391dd89d-6617-4212-aab7-a94d3c4c617f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"129a7fcc8503ce8949910162de1fa823d9370d6999b79ca9d6e3b1204eee54e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"97b54c62c879084f09800d76b8478521b07e47adf303c9bf8aa4fb3023c81fe2","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-9pv22x\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e350f5c8100\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-9pv22x: failed waiting for cluster=\"cp-ystream-upgrade-5-0-9pv22x\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-xm47zptwcsjh\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead\",\n \"name\": \"cbc721e9-e4bf-4464-8b12-84eda9921ead\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:43.593514078Z\",\n \"endTime\": \"2026-10-16T11:47:49.853897009Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead\n correlation ID: d3fa4fbf-6b99-46ed-95f4-434c377ebaea\n client request ID: 4804a3ed-75bb-478f-aa09-5c0568252a82\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-9pv22x\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x2e350f5c8100\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-9pv22x: failed waiting for cluster=\"cp-ystream-upgrade-5-0-9pv22x\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-xm47zptwcsjh\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead\",\n \"name\": \"cbc721e9-e4bf-4464-8b12-84eda9921ead\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:43.593514078Z\",\n \"endTime\": \"2026-10-16T11:47:49.853897009Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbc721e9-e4bf-4464-8b12-84eda9921ead\n correlation ID: d3fa4fbf-6b99-46ed-95f4-434c377ebaea\n client request ID: 4804a3ed-75bb-478f-aa09-5c0568252a82\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"12e0908181358cd5fac969d88ab4effc93196df9d74daf0c07998aec33a1e660","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"0040558a56ca2b78035a7b68cf46efa81a3757a14f72084b294057ecc8885c49","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-9jbnpswbqhnc/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-9jbnpswbqhnc/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-9jbnpswbqhnc/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"1604fb7e44321019a44224febbd026dce9f8034c5c615e09741ce4e20b21f7c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"02885cebd9f78ad3acdecfda8dfeea89bd33beff91262b016a2ea9791fecec67","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cd963448a00\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-g99wvvnb2gqt\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\",\n \"name\": \"feb6dbe3-4518-4432-8ee1-e82bfe257e23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:40.122769052Z\",\n \"endTime\": \"2026-10-16T11:47:49.816923189Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\n correlation ID: 6e26fa23-6c8f-48c7-84e4-93c4480408ac\n client request ID: 0de8699c-e23a-4388-aef8-31bf31c0384d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cd963448a00\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-g99wvvnb2gqt\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\",\n \"name\": \"feb6dbe3-4518-4432-8ee1-e82bfe257e23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:40.122769052Z\",\n \"endTime\": \"2026-10-16T11:47:49.816923189Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/feb6dbe3-4518-4432-8ee1-e82bfe257e23\n correlation ID: 6e26fa23-6c8f-48c7-84e4-93c4480408ac\n client request ID: 0de8699c-e23a-4388-aef8-31bf31c0384d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1824b54e35cccff223b2d7dd498fa7369000e69ca7e9ec1e400dcce653479ef6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"49b4d135392c1a8dfe2a53dd66eb8524c2043df0a329f9fd87409c7f19e72b70","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d54a9ec8640\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-gpfqm2n76z2f\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643\",\n \"name\": \"9319e8d3-f312-4003-b90c-21f9495a0643\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:40.951220038Z\",\n \"endTime\": \"2026-10-16T11:47:49.835941737Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643\n correlation ID: 7ab67cc1-868f-400c-b479-651074342b57\n client request ID: f0c8515f-2da0-4d17-b772-fbcda0b27f98\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d54a9ec8640\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-gpfqm2n76z2f\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643\",\n \"name\": \"9319e8d3-f312-4003-b90c-21f9495a0643\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:40.951220038Z\",\n \"endTime\": \"2026-10-16T11:47:49.835941737Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9319e8d3-f312-4003-b90c-21f9495a0643\n correlation ID: 7ab67cc1-868f-400c-b479-651074342b57\n client request ID: f0c8515f-2da0-4d17-b772-fbcda0b27f98\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1b14d213c57e1cb565adde03cddfd77a1e3d8b842a0981518e3775bd83cb4d17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"51a306129e11393c28658e5503ee4f1ab0f2982ff766d534a0cb44e56e7afb0d","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x30530c2049c0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pq2pqkkk7kkv\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1\",\n \"name\": \"e1c4fc61-b13a-4949-9525-c8e2995366d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.276521952Z\",\n \"endTime\": \"2026-10-16T11:47:34.729548419Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1\n correlation ID: cc4b631c-4cdd-46df-8676-8256af846370\n client request ID: b0546f64-7657-4fa7-8053-0510bf32e452\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x30530c2049c0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-pq2pqkkk7kkv\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1\",\n \"name\": \"e1c4fc61-b13a-4949-9525-c8e2995366d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.276521952Z\",\n \"endTime\": \"2026-10-16T11:47:34.729548419Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1c4fc61-b13a-4949-9525-c8e2995366d1\n correlation ID: cc4b631c-4cdd-46df-8676-8256af846370\n client request ID: b0546f64-7657-4fa7-8053-0510bf32e452\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1cb7c877a799afeebf1b3c487c963334a231e390e51fe2bdf79f30f0a5232543","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"94a66c07cad70ee5eaeb4c1ee5f1c2f9726f80978ce10d86eaeb30122d478883","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-nlkpgh277c95/cluster-candidate-4-23-n82d96 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1649075acb60\u003e: \n failed to create HCP cluster cluster-candidate-4-23-n82d96: failed waiting for cluster=\"cluster-candidate-4-23-n82d96\" in resourcegroup=\"rg-candidate-4-23-nlkpgh277c95\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3\",\n \"name\": \"27de5df4-c265-4de7-97d7-ba19926470a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.252038061Z\",\n \"endTime\": \"2026-10-16T11:47:35.485949636Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3\n correlation ID: 375617cf-8ae4-4fb0-8226-a993b64c2efa\n client request ID: 1d6dd6d4-ca93-447b-9e62-a692550d6007\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-nlkpgh277c95/cluster-candidate-4-23-n82d96 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x1649075acb60\u003e: \n failed to create HCP cluster cluster-candidate-4-23-n82d96: failed waiting for cluster=\"cluster-candidate-4-23-n82d96\" in resourcegroup=\"rg-candidate-4-23-nlkpgh277c95\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3\",\n \"name\": \"27de5df4-c265-4de7-97d7-ba19926470a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.252038061Z\",\n \"endTime\": \"2026-10-16T11:47:35.485949636Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27de5df4-c265-4de7-97d7-ba19926470a3\n correlation ID: 375617cf-8ae4-4fb0-8226-a993b64c2efa\n client request ID: 1d6dd6d4-ca93-447b-9e62-a692550d6007\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1dabe77a5308dbf70aca83d1f753514f0d65c2b11e23857604ceb26aa4992e3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"ec0f46269a3e3f9e6756c36a09c2cdcdf54acfc291e0b485ea33cb3efad67bc3","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc7fcee78400\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-xkdmlz6n9jw9\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035\",\n \"name\": \"9437713a-127b-49bb-b254-8e11c4cea035\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:19:30.237979438Z\",\n \"endTime\": \"2026-10-16T11:48:39.854496595Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035\n correlation ID: c47ad550-f9bc-4b54-9fb4-b9c071da2e2e\n client request ID: 7908e69f-5b28-487e-b354-e80c3f6a203f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc7fcee78400\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-xkdmlz6n9jw9\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035\",\n \"name\": \"9437713a-127b-49bb-b254-8e11c4cea035\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:19:30.237979438Z\",\n \"endTime\": \"2026-10-16T11:48:39.854496595Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9437713a-127b-49bb-b254-8e11c4cea035\n correlation ID: c47ad550-f9bc-4b54-9fb4-b9c071da2e2e\n client request ID: 7908e69f-5b28-487e-b354-e80c3f6a203f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"211cb61ac748d7b68b4a69d6bec8bd467563a4c1cdfc5ad0573a654927dccb08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"alert","signature_id":"e2f6ed75b3c3a7f5c74c0c161a3ab71c8a98b8f056b27a66eea5259563bedf3c","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterDeletionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T12:43:13Z\n Ended: 2026-10-16T13:05:11Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterDeletionErrors6h30m\", cluster=\"ci01-j9203200-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-deletion-errors\"\n Description: More than 30% of completed cluster delete operations on ci01-j9203200-svc failed or were canceled over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9e7d58cf80cefdbf58d438cd330cbadc1e097f5d57f87fa6133331546f9b5534","failure_pattern":"alert [svc] userJourneyClusterDeletionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterDeletionErrors6h30m fired"}},{"row_id":"2296134d7510af6f917a12870aab9cab424302e36bdfdf1b0a1a05ce60dfeb32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"d4eef89b2cf8249a4c0863dbed753ac1980c9cb8463f4f58576a4a7e55a4fc59","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x2618dbece140\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-j5vlvmgjsqjz\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645\",\n \"name\": \"a389c8be-8768-4995-ad5b-c321a6398645\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:56.786424151Z\",\n \"endTime\": \"2026-10-16T11:47:59.869771907Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645\n correlation ID: a9be42e7-990f-46ed-82b2-614c0f5c93e4\n client request ID: f1d3955d-3265-447b-966c-cfe8c54031e0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0x2618dbece140\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-j5vlvmgjsqjz\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645\",\n \"name\": \"a389c8be-8768-4995-ad5b-c321a6398645\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:56.786424151Z\",\n \"endTime\": \"2026-10-16T11:47:59.869771907Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a389c8be-8768-4995-ad5b-c321a6398645\n correlation ID: a9be42e7-990f-46ed-82b2-614c0f5c93e4\n client request ID: f1d3955d-3265-447b-966c-cfe8c54031e0\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"284118b9335c31522af4daff03f4ede36f5cfa1df02b3f676a43a7ba6bb9ac62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"42fa5aa1fc3c312b35f41fb9e8f7ab9f22d0742794def38b6d37010c17c73253","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xeb24e5fc1c0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-s7wwcnxg7nr6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff\",\n \"name\": \"bddfb7c0-8782-439a-a29c-5d984838d4ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:30.158724197Z\",\n \"endTime\": \"2026-10-16T12:43:31.12042993Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff\n correlation ID: 76f022c5-91e3-4c33-8b07-8710b12f4b32\n client request ID: 6e812c92-cfd2-484b-9f9d-c35d3bef9df7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xeb24e5fc1c0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-s7wwcnxg7nr6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff\",\n \"name\": \"bddfb7c0-8782-439a-a29c-5d984838d4ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:30.158724197Z\",\n \"endTime\": \"2026-10-16T12:43:31.12042993Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bddfb7c0-8782-439a-a29c-5d984838d4ff\n correlation ID: 76f022c5-91e3-4c33-8b07-8710b12f4b32\n client request ID: 6e812c92-cfd2-484b-9f9d-c35d3bef9df7\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2a75f3325ed05f7e2e163d4e1a60a31c1e3eb00ac4434980e5f9156de280f04f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"977474b948f7cd6d6e0c39010a84ec7957861d87ffc87786f2fb4bc1065ae482","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-k8896m6vhvph/cluster-candidate-5-0-svffqc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x4408644c620\u003e: \n failed to create HCP cluster cluster-candidate-5-0-svffqc: failed waiting for cluster=\"cluster-candidate-5-0-svffqc\" in resourcegroup=\"rg-candidate-5-0-k8896m6vhvph\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc\",\n \"name\": \"f521b84d-3184-4232-9fe7-cd33b6490fcc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.238638011Z\",\n \"endTime\": \"2026-10-16T11:47:34.761578844Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc\n correlation ID: edad4ec0-1e19-4f0a-84a2-f8ef2fb08495\n client request ID: 6ec90bea-9fe0-4677-b51f-03ee25f8acb9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-k8896m6vhvph/cluster-candidate-5-0-svffqc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x4408644c620\u003e: \n failed to create HCP cluster cluster-candidate-5-0-svffqc: failed waiting for cluster=\"cluster-candidate-5-0-svffqc\" in resourcegroup=\"rg-candidate-5-0-k8896m6vhvph\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc\",\n \"name\": \"f521b84d-3184-4232-9fe7-cd33b6490fcc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.238638011Z\",\n \"endTime\": \"2026-10-16T11:47:34.761578844Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521b84d-3184-4232-9fe7-cd33b6490fcc\n correlation ID: edad4ec0-1e19-4f0a-84a2-f8ef2fb08495\n client request ID: 6ec90bea-9fe0-4677-b51f-03ee25f8acb9\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"31f33524374e4ad54b6b08d5e078d8a054d92e71d25725e94300340d0fc878fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"396888b5a0686222585c428a9bce3005511dca83925bb8f3900a64c106989f35","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kmzj2j with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x163f9a6e2b60\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-kmzj2j: failed waiting for cluster=\"np-version-upgrade-cluster-kmzj2j\" in resourcegroup=\"rg-np-version-upgrade-kmzj2j-88dfmkbc2kcf\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff\",\n \"name\": \"3112585c-f06a-4eed-8bc2-c5766d255eff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:30.034396275Z\",\n \"endTime\": \"2026-10-16T12:43:31.11907947Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff\n correlation ID: 61a6bffb-48df-441c-9977-a498349294c0\n client request ID: 92258152-5b22-402e-88a2-d9f0a0d3f850\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kmzj2j with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x163f9a6e2b60\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-kmzj2j: failed waiting for cluster=\"np-version-upgrade-cluster-kmzj2j\" in resourcegroup=\"rg-np-version-upgrade-kmzj2j-88dfmkbc2kcf\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff\",\n \"name\": \"3112585c-f06a-4eed-8bc2-c5766d255eff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:30.034396275Z\",\n \"endTime\": \"2026-10-16T12:43:31.11907947Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3112585c-f06a-4eed-8bc2-c5766d255eff\n correlation ID: 61a6bffb-48df-441c-9977-a498349294c0\n client request ID: 92258152-5b22-402e-88a2-d9f0a0d3f850\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"32fecd8f02896f4f60613c81627b5ac22b77d2a30520667c948d09638bf9dd7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"380c037c9544119d3f0074b1f72fc443bdb997e65ae25e66e8645ec89ab34ab3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"4dcc9eff48755be1b1a70a880ea55753d6810313149150fb1254bf0ca5907e39","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x57a86294180\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wqj2khzntl8h\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a\",\n \"name\": \"a00835a9-2946-492c-83cf-4feb60e97d6a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:41.970124073Z\",\n \"endTime\": \"2026-10-16T11:47:49.845444522Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a\n correlation ID: b829a774-d420-42d1-910f-558c24c0e81b\n client request ID: fa3b6c1d-d6ce-4bc1-818f-7e083d5113d0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x57a86294180\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wqj2khzntl8h\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a\",\n \"name\": \"a00835a9-2946-492c-83cf-4feb60e97d6a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:41.970124073Z\",\n \"endTime\": \"2026-10-16T11:47:49.845444522Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a00835a9-2946-492c-83cf-4feb60e97d6a\n correlation ID: b829a774-d420-42d1-910f-558c24c0e81b\n client request ID: fa3b6c1d-d6ce-4bc1-818f-7e083d5113d0\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3a72e1b3a431ae8b4f28bb66fc4cd59b13971a540d8f3ae4e7f631953515adba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"e47f272ea8d6238d4737d12adebe1d3669788d8b3f1a6689760a033749bf48c8","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e5b5b4ea680\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-pqhbfg7fld4c\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f\",\n \"name\": \"934784ee-76b6-4805-b96a-804962fd4f8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:14.866312322Z\",\n \"endTime\": \"2026-10-16T11:47:15.217711166Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f\n correlation ID: 0f7d9c8a-342c-410a-bac7-fcbf38991b8b\n client request ID: e542720e-5ece-4d3c-a284-7d8bdebc13b0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e5b5b4ea680\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-pqhbfg7fld4c\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f\",\n \"name\": \"934784ee-76b6-4805-b96a-804962fd4f8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:14.866312322Z\",\n \"endTime\": \"2026-10-16T11:47:15.217711166Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/934784ee-76b6-4805-b96a-804962fd4f8f\n correlation ID: 0f7d9c8a-342c-410a-bac7-fcbf38991b8b\n client request ID: e542720e-5ece-4d3c-a284-7d8bdebc13b0\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"462d2a8adf482db045a36556cb70a7035e54cbfec537734ff7b55e5d93694a24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"eef2dbac73ba59a5bfe3c29d485600cdcc6126efca87a76b6b356a29aecedab8","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xbcb4282a180\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-x5pjtlmznn57\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f\",\n \"name\": \"6909c02f-59ad-446d-af3d-48126d54318f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:41.793037251Z\",\n \"endTime\": \"2026-10-16T11:47:49.840680018Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f\n correlation ID: 6249e6eb-e2f2-49a4-8b3d-fa8588c616f1\n client request ID: fcd7ba19-a355-4a1f-b414-c70f5d773a28\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xbcb4282a180\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-x5pjtlmznn57\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f\",\n \"name\": \"6909c02f-59ad-446d-af3d-48126d54318f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:41.793037251Z\",\n \"endTime\": \"2026-10-16T11:47:49.840680018Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6909c02f-59ad-446d-af3d-48126d54318f\n correlation ID: 6249e6eb-e2f2-49a4-8b3d-fa8588c616f1\n client request ID: fcd7ba19-a355-4a1f-b414-c70f5d773a28\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4999a659311aa8ad4f41911a46420cd1a46a07b7e42f2904a8b119be03d52761","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"2bee3baecdf764b673e8d30ec79e81ae8ad2ba4a2f7c7205567bf951a79600bc","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-js4rfm\" with version 4.20.42 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a1f4bafcbc0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-js4rfm: failed waiting for cluster=\"cluster-zstream-4-20-js4rfm\" in resourcegroup=\"rg-zstream-upgrade-4-20-r2p2qj59lsd4\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f\",\n \"name\": \"862c5703-2680-4bf6-9df4-c4097e46e34f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:46.295357285Z\",\n \"endTime\": \"2026-10-16T11:47:49.823361811Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f\n correlation ID: 38689985-8663-45c2-b6d3-3c921b0b9d8e\n client request ID: de817d4d-06c3-4aa8-aeb2-34912bfe8ec1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-js4rfm\" with version 4.20.42 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a1f4bafcbc0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-js4rfm: failed waiting for cluster=\"cluster-zstream-4-20-js4rfm\" in resourcegroup=\"rg-zstream-upgrade-4-20-r2p2qj59lsd4\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f\",\n \"name\": \"862c5703-2680-4bf6-9df4-c4097e46e34f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:46.295357285Z\",\n \"endTime\": \"2026-10-16T11:47:49.823361811Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/862c5703-2680-4bf6-9df4-c4097e46e34f\n correlation ID: 38689985-8663-45c2-b6d3-3c921b0b9d8e\n client request ID: de817d4d-06c3-4aa8-aeb2-34912bfe8ec1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4efe59be6730c8c8460db9510af7c4869bb756e46680c5fa76c433bb0224dff6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"3f0e76f754f9f39ad23ff397a68c8bda34243b425a937395a42136cdfada3fde","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1edc8262aa20\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6p6j6qsmdwpj\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7\",\n \"name\": \"4a5c3106-eb2a-45eb-a602-c5ec51db82a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.696535718Z\",\n \"endTime\": \"2026-10-16T11:47:34.770284565Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7\n correlation ID: faeb3822-724b-4825-a824-edb8c1b845c1\n client request ID: 51d01eb4-edcc-4337-ad97-569b5b96ddb2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1edc8262aa20\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6p6j6qsmdwpj\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7\",\n \"name\": \"4a5c3106-eb2a-45eb-a602-c5ec51db82a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.696535718Z\",\n \"endTime\": \"2026-10-16T11:47:34.770284565Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a5c3106-eb2a-45eb-a602-c5ec51db82a7\n correlation ID: faeb3822-724b-4825-a824-edb8c1b845c1\n client request ID: 51d01eb4-edcc-4337-ad97-569b5b96ddb2\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"617d827581669d7a75edcd50a54bbd5bfb5840ed673bb9ec8979f0c447feffb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"278afa23556e267f104e90f63791a567ee1b91c5dae6d332f1391b922d1ce932","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:126]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xaa8af49c580\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-zsmzg99h2cph\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6\",\n \"name\": \"a4151c0d-6fe7-46e7-ae06-85bca0bbfda6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:19:28.3068615Z\",\n \"endTime\": \"2026-10-16T11:48:29.818041797Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6\n correlation ID: 5e6c1f91-f464-4ac3-8b66-a7151ccf033d\n client request ID: 1763d574-e139-47a7-a99c-8d752034afa4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:126]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xaa8af49c580\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-zsmzg99h2cph\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6\",\n \"name\": \"a4151c0d-6fe7-46e7-ae06-85bca0bbfda6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:19:28.3068615Z\",\n \"endTime\": \"2026-10-16T11:48:29.818041797Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a4151c0d-6fe7-46e7-ae06-85bca0bbfda6\n correlation ID: 5e6c1f91-f464-4ac3-8b66-a7151ccf033d\n client request ID: 1763d574-e139-47a7-a99c-8d752034afa4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"643aec3e850c2ac7932bf82b387f69e7602d5d889a266a7bfb574417e943225f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"721ede543ee7748f0c12fb1646a03204d96825f09741021d26520978ddc98f51","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x2cc9b8996a60\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-w5rdt4gkt25d\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b\",\n \"name\": \"1a2ea3fa-8f68-40b2-a65c-a717ffa2902b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:36.212608814Z\",\n \"endTime\": \"2026-10-16T11:47:39.813722246Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b\n correlation ID: a01e533d-97b0-4d30-8c31-a1730fbb9c1b\n client request ID: dd83ab35-b0c7-4853-b865-900c9dc97b10\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0x2cc9b8996a60\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-w5rdt4gkt25d\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b\",\n \"name\": \"1a2ea3fa-8f68-40b2-a65c-a717ffa2902b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:36.212608814Z\",\n \"endTime\": \"2026-10-16T11:47:39.813722246Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a2ea3fa-8f68-40b2-a65c-a717ffa2902b\n correlation ID: a01e533d-97b0-4d30-8c31-a1730fbb9c1b\n client request ID: dd83ab35-b0c7-4853-b865-900c9dc97b10\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"657561e022c5649ad19c4f72d7c8b562c943e469598275121a744b6e33d06001","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"187cdd3ccfea7e8b9a553c281602c3a4ea6ca7108b24fd37be5a95399b1285a3","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-x5jpgh\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ba9f4280b60\u003e: \n failed to create HCP cluster np-noedge-x5jpgh: failed waiting for cluster=\"np-noedge-x5jpgh\" in resourcegroup=\"rg-np-noedge-x5jpgh-nmz7rtgztcdt\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2\",\n \"name\": \"a63b7a0e-c2ff-4ea5-ad50-a2392535dea2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:28.980964675Z\",\n \"endTime\": \"2026-10-16T12:43:31.119616474Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2\n correlation ID: a6626179-2136-45bb-825a-5b925656e4cd\n client request ID: 28e2874d-4a4a-42c0-9201-66d388c353c7\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-x5jpgh\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ba9f4280b60\u003e: \n failed to create HCP cluster np-noedge-x5jpgh: failed waiting for cluster=\"np-noedge-x5jpgh\" in resourcegroup=\"rg-np-noedge-x5jpgh-nmz7rtgztcdt\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2\",\n \"name\": \"a63b7a0e-c2ff-4ea5-ad50-a2392535dea2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:28.980964675Z\",\n \"endTime\": \"2026-10-16T12:43:31.119616474Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a63b7a0e-c2ff-4ea5-ad50-a2392535dea2\n correlation ID: a6626179-2136-45bb-825a-5b925656e4cd\n client request ID: 28e2874d-4a4a-42c0-9201-66d388c353c7\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6691f529110b512a293a61610d56d8e617f3d6df9e942242f05eca49db4991ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"alert","signature_id":"3c7900addeb947a4a4e739b2d41a366679056acc6cfdec3d613c346f9cc41807","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T11:31:52Z\n Ended: 2026-10-16T11:54:49Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j9203200-svc\", component=\"backend\", name=\"identityroleassignments\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue identityroleassignments has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T11:39:48Z\n Ended: 2026-10-16T11:54:49Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j9203200-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue dataplaneoidcfederation has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-16T12:07:49Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j9203200-svc\", component=\"backend\", name=\"ensuremanagedresourcegroup\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue ensuremanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"6bd5a3f17bacc6e5f10458949f178036c59d43a9603f581f34e49f98e9dcf2fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"41b4e4ab1b6b9f7c1850160dbccf81ec34d905cb168826f78de485a244e050f8","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x349580025280\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xbkhmb99h5bw\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132\",\n \"name\": \"56bd2ed4-8b90-4d1e-9b32-b3e975d36132\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:14.895598807Z\",\n \"endTime\": \"2026-10-16T11:47:14.891231174Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132\n correlation ID: 44a81805-e64c-4ba6-9409-983e8b170312\n client request ID: 0115f71b-c2d8-4f13-b17d-9fb8699799fd\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x349580025280\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xbkhmb99h5bw\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132\",\n \"name\": \"56bd2ed4-8b90-4d1e-9b32-b3e975d36132\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:14.895598807Z\",\n \"endTime\": \"2026-10-16T11:47:14.891231174Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd2ed4-8b90-4d1e-9b32-b3e975d36132\n correlation ID: 44a81805-e64c-4ba6-9409-983e8b170312\n client request ID: 0115f71b-c2d8-4f13-b17d-9fb8699799fd\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6c2cc82fbc207732b0b2e082c60c94e23c3f1dba4fc962e347af5fd92c8829b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"0a4b69efc91d14544bf2507c26816357641f81a32c9c58337f77d325cca61d19","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x123425a4b400\u003e: \n failed to create HCP cluster np-downgrade-8lhgvh: failed waiting for cluster=\"np-downgrade-8lhgvh\" in resourcegroup=\"rg-np-downgrade-8lhgvh-d55cjbncvrzk\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9\",\n \"name\": \"c6e38d64-f6cf-46fa-8335-12cd44b337a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:28.696987476Z\",\n \"endTime\": \"2026-10-16T12:43:31.118819677Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9\n correlation ID: 433e0cbf-f087-4fdc-b83e-56c9b436990e\n client request ID: a497577e-eb3d-4aa3-a690-f0283dedfcd8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x123425a4b400\u003e: \n failed to create HCP cluster np-downgrade-8lhgvh: failed waiting for cluster=\"np-downgrade-8lhgvh\" in resourcegroup=\"rg-np-downgrade-8lhgvh-d55cjbncvrzk\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9\",\n \"name\": \"c6e38d64-f6cf-46fa-8335-12cd44b337a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:28.696987476Z\",\n \"endTime\": \"2026-10-16T12:43:31.118819677Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6e38d64-f6cf-46fa-8335-12cd44b337a9\n correlation ID: 433e0cbf-f087-4fdc-b83e-56c9b436990e\n client request ID: a497577e-eb3d-4aa3-a690-f0283dedfcd8\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"741eb0f83228f134cdc05dea8a6ac6ff75ad57040ad990638cd27472da4b1711","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"alert","signature_id":"5c8ff5a4b50c3b1243adb8b3603e526fd1b9318ecd34f70e7fdaab4ed1afa718","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-16T12:19:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j9203200-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of completed cluster create (install) operations on ci01-j9203200-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired"}},{"row_id":"745cba2d903d2c445b15b9334ed695da49a7d4500d4887db8872702d60360bb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"ce4b2abc9b3ceb68bbd021867c606e5a5cb6e1e7afd106b40ea6fb89ee7a23a6","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x32fe88162120\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-pjd2p5pkfgjp\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee\",\n \"name\": \"3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:42.802740388Z\",\n \"endTime\": \"2026-10-16T11:47:49.838674727Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee\n correlation ID: 368abfde-4c93-4713-8780-51b84de0d719\n client request ID: c27bf18a-f74b-4aba-9e88-1de4311a6a7c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x32fe88162120\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-pjd2p5pkfgjp\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee\",\n \"name\": \"3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:42.802740388Z\",\n \"endTime\": \"2026-10-16T11:47:49.838674727Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e475ab3-8ca4-4ffa-8ab7-45aae534d7ee\n correlation ID: 368abfde-4c93-4713-8780-51b84de0d719\n client request ID: c27bf18a-f74b-4aba-9e88-1de4311a6a7c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"766367ab39cc1993087554940078e099867cdd09e2f67d9f27e8922b718520e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"05538f072fdbd9e2a7d3c80b025b5f0230de28b40a21437ec1e51553c1f9bd1e","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x390e05a68de0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xqknpzfzmr5w\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\",\n \"name\": \"73d23223-112b-4b7f-ba15-3c3397a86d65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:15.514720241Z\",\n \"endTime\": \"2026-10-16T11:47:24.729519268Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\n correlation ID: 88facd09-aef2-4a7f-9c7e-db87f515aac5\n client request ID: 4c75b22c-aded-487f-a952-f949adbe49c6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:163]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0x390e05a68de0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xqknpzfzmr5w\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\",\n \"name\": \"73d23223-112b-4b7f-ba15-3c3397a86d65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:15.514720241Z\",\n \"endTime\": \"2026-10-16T11:47:24.729519268Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/73d23223-112b-4b7f-ba15-3c3397a86d65\n correlation ID: 88facd09-aef2-4a7f-9c7e-db87f515aac5\n client request ID: 4c75b22c-aded-487f-a952-f949adbe49c6\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"767584734c8186a5bfb0e5f9d0abaf2e419f625592c7420a220c3b66f917cb92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"f4be36cc0230ad342fd316be591c2129d772f7846989d11fc93f73065bc1f045","test_name":"Fleet should have HCP resource requirements data","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.000s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:\nDataCurrent condition must be True\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.000s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:105 with:\nDataCurrent condition must be True\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96d801deab41ec98748c7082577ee746091d92728a5e786bd4eef6a4d1b2ca0f","failure_pattern":"DataCurrent condition must be True","search_query":"DataCurrent condition must be True"}},{"row_id":"7c2ed652c078a665ce9472172c9576652895a008c6f4489e61338a7fa4b3ee58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"518f777d390e10ac4956e3dca8c437996930936e6b6c2b05b130888e0b602e46","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xe1af249a880\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6jlkrz7lq599\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f\",\n \"name\": \"634b8f7c-5c70-49a7-b19e-48bd89488e6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:15.833452215Z\",\n \"endTime\": \"2026-10-16T11:47:15.218786602Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f\n correlation ID: b56813d3-03b1-46c4-a01a-7aa714d3bb3b\n client request ID: 6bba57fa-9f7a-491b-9479-621cd45e5270\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xe1af249a880\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6jlkrz7lq599\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f\",\n \"name\": \"634b8f7c-5c70-49a7-b19e-48bd89488e6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:15.833452215Z\",\n \"endTime\": \"2026-10-16T11:47:15.218786602Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/634b8f7c-5c70-49a7-b19e-48bd89488e6f\n correlation ID: b56813d3-03b1-46c4-a01a-7aa714d3bb3b\n client request ID: 6bba57fa-9f7a-491b-9479-621cd45e5270\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"817ab78517e4a128b74d874a42bf34825236f1320589262fc19d4b184e98b71c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"d5f86ee2aac4ee5b3ef45df0c2a654bb28ac6de1273c99cc3362c3b527063993","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x26ae0b1187a0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-98xd6plvc54r\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a\",\n \"name\": \"ef85c04a-5d49-4fdd-934c-fb1c0925da5a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:51.520890154Z\",\n \"endTime\": \"2026-10-16T11:47:59.828901888Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a\n correlation ID: 61b08897-2788-477c-9e28-af5643f8731a\n client request ID: f65210df-702d-4b5f-bf36-5cb3949c8b60\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0x26ae0b1187a0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-98xd6plvc54r\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a\",\n \"name\": \"ef85c04a-5d49-4fdd-934c-fb1c0925da5a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:51.520890154Z\",\n \"endTime\": \"2026-10-16T11:47:59.828901888Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef85c04a-5d49-4fdd-934c-fb1c0925da5a\n correlation ID: 61b08897-2788-477c-9e28-af5643f8731a\n client request ID: f65210df-702d-4b5f-bf36-5cb3949c8b60\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"89b94fe0ba4e6c270bb0132e94391aee438350312f57ca16b2677b8d188ff7f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"0fd93a4c34a4de4e23345b99b01e77944e8813b5762ee29651e30a253ad3a392","test_name":"SRE starts a control plane z-stream upgrade with an SRE pin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster \"sre-version-pin-bktv28\" at 4.21.37\nUnexpected error:\n \u003c*fmt.wrapError | 0x28142a660f40\u003e: \n failed to create HCP cluster sre-version-pin-bktv28: failed waiting for cluster=\"sre-version-pin-bktv28\" in resourcegroup=\"admin-api-version-pin-nqqcn8wlvf5c\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031\",\n \"name\": \"b2b04de1-00d3-4f87-be7d-90b2c747d031\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:41.261950808Z\",\n \"endTime\": \"2026-10-16T11:47:49.830086043Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031\n correlation ID: c9fa32a4-a67f-4d74-87e4-0703a6dd15ef\n client request ID: 4b87fe24-0400-4a09-8cfa-9dbfa250ce81\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:114]: failed to create cluster \"sre-version-pin-bktv28\" at 4.21.37\nUnexpected error:\n \u003c*fmt.wrapError | 0x28142a660f40\u003e: \n failed to create HCP cluster sre-version-pin-bktv28: failed waiting for cluster=\"sre-version-pin-bktv28\" in resourcegroup=\"admin-api-version-pin-nqqcn8wlvf5c\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031\",\n \"name\": \"b2b04de1-00d3-4f87-be7d-90b2c747d031\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:41.261950808Z\",\n \"endTime\": \"2026-10-16T11:47:49.830086043Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b2b04de1-00d3-4f87-be7d-90b2c747d031\n correlation ID: c9fa32a4-a67f-4d74-87e4-0703a6dd15ef\n client request ID: 4b87fe24-0400-4a09-8cfa-9dbfa250ce81\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8f68101474a7da27d2366f42e7e634bf13095f7c2e99efdcd7289ecb2ef0e2b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"057841c6d731f163f47fffd09f49fa90b912e57433629c7df3d0e0334e9e5e9c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-g5rdgl7zwjl6/cluster-candidate-4-20-k9b9wf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x18f77dd205c0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-k9b9wf: failed waiting for cluster=\"cluster-candidate-4-20-k9b9wf\" in resourcegroup=\"rg-candidate-4-20-g5rdgl7zwjl6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\",\n \"name\": \"0984890f-0a19-47d9-9a1c-275d44c6e314\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:36.959680763Z\",\n \"endTime\": \"2026-10-16T11:47:39.816204138Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\n correlation ID: c1d2f06b-ddfc-4fd2-8766-74fa80998c3b\n client request ID: ef5cac13-1a05-485d-b15a-d283225aac9d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-g5rdgl7zwjl6/cluster-candidate-4-20-k9b9wf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x18f77dd205c0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-k9b9wf: failed waiting for cluster=\"cluster-candidate-4-20-k9b9wf\" in resourcegroup=\"rg-candidate-4-20-g5rdgl7zwjl6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\",\n \"name\": \"0984890f-0a19-47d9-9a1c-275d44c6e314\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:36.959680763Z\",\n \"endTime\": \"2026-10-16T11:47:39.816204138Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0984890f-0a19-47d9-9a1c-275d44c6e314\n correlation ID: c1d2f06b-ddfc-4fd2-8766-74fa80998c3b\n client request ID: ef5cac13-1a05-485d-b15a-d283225aac9d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"98284d2ead368e7e7ac135d27249b8c7b95a15eab18a7eb4f367a9ea617ce61e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"6b59025aff4632ced9c54e21a7d8ba849c8f13e1bb860284550175feaf03cd58","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-8xw5fg\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x370f8d648c60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-8xw5fg: failed waiting for cluster=\"cp-ystream-upgrade-4-23-8xw5fg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-b5xx2gk9qwbp\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f\",\n \"name\": \"56bd1583-21e2-4843-b05f-31ac6826444f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:33.267640725Z\",\n \"endTime\": \"2026-10-16T11:47:35.486996122Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f\n correlation ID: 3c2cdb81-95c2-4c03-9b23-7f4468666f93\n client request ID: ca21a34b-5eb9-4e2d-9400-6431d58f4052\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-8xw5fg\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x370f8d648c60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-8xw5fg: failed waiting for cluster=\"cp-ystream-upgrade-4-23-8xw5fg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-b5xx2gk9qwbp\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f\",\n \"name\": \"56bd1583-21e2-4843-b05f-31ac6826444f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:33.267640725Z\",\n \"endTime\": \"2026-10-16T11:47:35.486996122Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56bd1583-21e2-4843-b05f-31ac6826444f\n correlation ID: 3c2cdb81-95c2-4c03-9b23-7f4468666f93\n client request ID: ca21a34b-5eb9-4e2d-9400-6431d58f4052\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9a6321a523bfd82ec7d5324a6995f8d0500c03372887354b97e027e611f18582","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"1b5b51cd57c129f4bd09991200713b69d34af14417d439c9d03facc0f7809f38","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-zmn6qg at version 4.22.18\nUnexpected error:\n \u003c*fmt.wrapError | 0x8551e630ea0\u003e: \n failed to create HCP cluster np-skip-minor-zmn6qg: failed waiting for cluster=\"np-skip-minor-zmn6qg\" in resourcegroup=\"rg-np-skip-minor-zmn6qg-zq4qtnxjmlj6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e\",\n \"name\": \"99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:29.343845082Z\",\n \"endTime\": \"2026-10-16T12:43:31.120076578Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e\n correlation ID: ae7f1a1f-77db-45da-834f-d7f445c5f80e\n client request ID: abc08cab-9ae2-473b-a212-d5cf27cc3368\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-zmn6qg at version 4.22.18\nUnexpected error:\n \u003c*fmt.wrapError | 0x8551e630ea0\u003e: \n failed to create HCP cluster np-skip-minor-zmn6qg: failed waiting for cluster=\"np-skip-minor-zmn6qg\" in resourcegroup=\"rg-np-skip-minor-zmn6qg-zq4qtnxjmlj6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e\",\n \"name\": \"99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:29.343845082Z\",\n \"endTime\": \"2026-10-16T12:43:31.120076578Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/99a61900-3cfd-46fb-9fa1-09fe9c6b5f0e\n correlation ID: ae7f1a1f-77db-45da-834f-d7f445c5f80e\n client request ID: abc08cab-9ae2-473b-a212-d5cf27cc3368\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9d5964a24cbc1cb0c1fdbd2049dac0580e3041a79fd5faee579b0e0a906bb3f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"cf1cbb296deea632da0aa91ba2e9058c868e017af626fd893c179b4c9f1c65c1","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xb4e8f680740\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-6jkf7zpl5td4\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1\",\n \"name\": \"19c9efd8-d627-4375-9ef4-9babf234f8d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:14.041145923Z\",\n \"endTime\": \"2026-10-16T11:47:14.862669197Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1\n correlation ID: 9c949c6a-68de-4443-a83a-d888a74b990c\n client request ID: 4d47ea32-d094-469c-af18-9dd47dff7a28\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xb4e8f680740\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-6jkf7zpl5td4\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1\",\n \"name\": \"19c9efd8-d627-4375-9ef4-9babf234f8d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:14.041145923Z\",\n \"endTime\": \"2026-10-16T11:47:14.862669197Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19c9efd8-d627-4375-9ef4-9babf234f8d1\n correlation ID: 9c949c6a-68de-4443-a83a-d888a74b990c\n client request ID: 4d47ea32-d094-469c-af18-9dd47dff7a28\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9fa606d678c907a7311a56e0fbee3a37612a807ca21360958d561f094d647161","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"alert","signature_id":"bdc267b19eb19dceb0fc582a5893f5fe99d59172eb52ea317ffe1c61b8ab9c7e","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterDeletionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-16T13:12:08Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterDeletionErrors1h5m\", cluster=\"ci01-j9203200-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-deletion-errors\"\n Description: More than 72% of completed cluster delete operations on ci01-j9203200-svc failed or were canceled over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6685a5a6bc3f263f200f46fbec1e8851219602ada5b75c61eac4345b058adb70","failure_pattern":"alert [svc] userJourneyClusterDeletionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterDeletionErrors1h5m fired"}},{"row_id":"a40c0f2e67f199ae32523283b3fa2c6e93a6cd6b428e3b60c5e4477ff21131ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"0dd2468d8f40ede925773d79aa00d7b2731ac739d0aec50490727e167d99e1ce","test_name":"Fleet should have scheduling data for ready management clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.000s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:\nstamp 1 scheduling requested CPU must be positive\nExpected\n \u003cbool\u003e: true\nto be false\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.000s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:156 with:\nstamp 1 scheduling requested CPU must be positive\nExpected\n \u003cbool\u003e: true\nto be false","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0cd69af8904472f73d329003c8c5db9d0e892958b3af0d73d1a1dee951ccff6d","failure_pattern":"scheduling requested CPU must be positive","search_query":"stamp 1 scheduling requested CPU must be positive"}},{"row_id":"a6689109ca97819f1031555fc41ef42401b60274180fad14fddcaf9a49781ffa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"97485c8b78318018cb7e1ff269311e2c04f020e9ccc2721c72cf0b76e8b10422","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-m2vs55vfxjl6/cluster-candidate-4-21-tkgb5h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e0a8449c940\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tkgb5h: failed waiting for cluster=\"cluster-candidate-4-21-tkgb5h\" in resourcegroup=\"rg-candidate-4-21-m2vs55vfxjl6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf\",\n \"name\": \"62fb450c-e7f1-4190-bb7f-226b4ee61acf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:15.933025884Z\",\n \"endTime\": \"2026-10-16T11:47:15.220182174Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf\n correlation ID: 47d3a69f-84e1-4042-80f0-1c8fbfff617f\n client request ID: f0b59b24-6a55-4893-8114-1d2b9df17303\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-m2vs55vfxjl6/cluster-candidate-4-21-tkgb5h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e0a8449c940\u003e: \n failed to create HCP cluster cluster-candidate-4-21-tkgb5h: failed waiting for cluster=\"cluster-candidate-4-21-tkgb5h\" in resourcegroup=\"rg-candidate-4-21-m2vs55vfxjl6\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf\",\n \"name\": \"62fb450c-e7f1-4190-bb7f-226b4ee61acf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:15.933025884Z\",\n \"endTime\": \"2026-10-16T11:47:15.220182174Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/62fb450c-e7f1-4190-bb7f-226b4ee61acf\n correlation ID: 47d3a69f-84e1-4042-80f0-1c8fbfff617f\n client request ID: f0b59b24-6a55-4893-8114-1d2b9df17303\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a8b45cbc6b55c111b2e90c495be68288f8cf0e2b12d42f7226faaf352680bc92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"alert","signature_id":"19b6ab86b15a2714ad1a515f08f8394ebbf9aba03e8c697d567a805fe6d0e44f","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T11:35:33Z\n Ended: 2026-10-16T11:56:31Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9203200-svc\", component=\"backend\", name=\"dataplaneoidcfederation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue dataplaneoidcfederation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T11:36:30Z\n Ended: 2026-10-16T11:56:31Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9203200-svc\", component=\"backend\", name=\"clusterdegradedaggregator\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterdegradedaggregator has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"add4c5d4667cb3b79ef0b10381e310f86d389c5d1713f171ed123c8fd33eb0ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"a55604cd1e12d3c298c747acbe1545adf0311ab0a9c1e30c8e1feeaa1450872c","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0x3829cd510c40\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-4gqdqvkfqdht\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3\",\n \"name\": \"43fd4377-f606-4d10-93db-f23555a1e1b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:19:08.543890305Z\",\n \"endTime\": \"2026-10-16T11:48:09.8244489Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3\n correlation ID: e1749404-9e54-474d-bcf9-84f47251ca01\n client request ID: 8cba2cc4-4919-4b95-82c2-d9427a0e27b0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:135]: failed to create HCP cluster des-encrypt\nUnexpected error:\n \u003c*fmt.wrapError | 0x3829cd510c40\u003e: \n failed to create HCP cluster des-encrypt: failed waiting for cluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-4gqdqvkfqdht\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3\",\n \"name\": \"43fd4377-f606-4d10-93db-f23555a1e1b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:19:08.543890305Z\",\n \"endTime\": \"2026-10-16T11:48:09.8244489Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/43fd4377-f606-4d10-93db-f23555a1e1b3\n correlation ID: e1749404-9e54-474d-bcf9-84f47251ca01\n client request ID: 8cba2cc4-4919-4b95-82c2-d9427a0e27b0\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b04b4e9e3bbbeb51ddad20ff680d2834d5fbbae31ad55e3957fd0dfefa588754","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"107ada83ebd0b578c73b8f47d78199af675df1d0228d381e1c3b379e802effee","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-f26jnj\" with version 4.21.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c652124c5c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-f26jnj: failed waiting for cluster=\"cluster-zstream-4-21-f26jnj\" in resourcegroup=\"rg-zstream-upgrade-4-21-xq96l5g2nl8g\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff\",\n \"name\": \"32f3a7f1-4008-4b7b-bf84-4f1342cb3fff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:15.444176059Z\",\n \"endTime\": \"2026-10-16T11:47:24.728940607Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff\n correlation ID: 83ad100d-6464-4314-bda4-bc37110e46f7\n client request ID: 3c78c9bb-d24a-481a-85ea-5be29a996155\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-f26jnj\" with version 4.21.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c652124c5c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-f26jnj: failed waiting for cluster=\"cluster-zstream-4-21-f26jnj\" in resourcegroup=\"rg-zstream-upgrade-4-21-xq96l5g2nl8g\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff\",\n \"name\": \"32f3a7f1-4008-4b7b-bf84-4f1342cb3fff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:15.444176059Z\",\n \"endTime\": \"2026-10-16T11:47:24.728940607Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/32f3a7f1-4008-4b7b-bf84-4f1342cb3fff\n correlation ID: 83ad100d-6464-4314-bda4-bc37110e46f7\n client request ID: 3c78c9bb-d24a-481a-85ea-5be29a996155\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b59c2f76a3fc56c255937dffb2dd1d7ca020f971714252850552f3eeec8527cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"5f52cde04677912060df6f64e39e81dc642c9922c6a6f3cd67da57388aaedb2e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-rskl6m\" with version 5.0.0-rc.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x323cfa71cc80\u003e: \n failed to create HCP cluster cluster-zstream-5-0-rskl6m: failed waiting for cluster=\"cluster-zstream-5-0-rskl6m\" in resourcegroup=\"rg-zstream-upgrade-5-0-sr6fb558zbkp\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe\",\n \"name\": \"3bec8b37-4b2a-458d-8cd1-1393433480fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:26.138664953Z\",\n \"endTime\": \"2026-10-16T11:47:34.774754546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe\n correlation ID: e09bf7d2-5b59-4164-bb62-51bfaa1958d8\n client request ID: dc11d72e-3bf9-4874-b3c8-0dfb9ea5435b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-rskl6m\" with version 5.0.0-rc.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x323cfa71cc80\u003e: \n failed to create HCP cluster cluster-zstream-5-0-rskl6m: failed waiting for cluster=\"cluster-zstream-5-0-rskl6m\" in resourcegroup=\"rg-zstream-upgrade-5-0-sr6fb558zbkp\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe\",\n \"name\": \"3bec8b37-4b2a-458d-8cd1-1393433480fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:26.138664953Z\",\n \"endTime\": \"2026-10-16T11:47:34.774754546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3bec8b37-4b2a-458d-8cd1-1393433480fe\n correlation ID: e09bf7d2-5b59-4164-bb62-51bfaa1958d8\n client request ID: dc11d72e-3bf9-4874-b3c8-0dfb9ea5435b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b8bde92a5ea64a0a6a8e115c178272668c39a5cbab84700c52975a6f18feb0f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"df7cfa109123f6b9d56e9533cf7581cd5ae0ca66c8a0568d1106a07da3e58237","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e9cd3298520\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-dxm54wcws6hs\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb\",\n \"name\": \"9314458c-cab4-4fbd-b0e7-c2c9434231fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:29.929706477Z\",\n \"endTime\": \"2026-10-16T12:43:31.118752307Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb\n correlation ID: 7a91ca7e-bafb-4c9a-b8db-f6e7ad6f8b52\n client request ID: 39bf8f20-42d5-4454-a015-9f651c57e1cb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e9cd3298520\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-dxm54wcws6hs\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb\",\n \"name\": \"9314458c-cab4-4fbd-b0e7-c2c9434231fb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:29.929706477Z\",\n \"endTime\": \"2026-10-16T12:43:31.118752307Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9314458c-cab4-4fbd-b0e7-c2c9434231fb\n correlation ID: 7a91ca7e-bafb-4c9a-b8db-f6e7ad6f8b52\n client request ID: 39bf8f20-42d5-4454-a015-9f651c57e1cb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bb480c9b9e246a3291c98705d1771a73d900370ba4cfbac01345bb1238865abd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"494d8441a417398603173f97d342c28db189843720bc98016b63a187b18c50ad","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f141cf40a40\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-s45bf5gctwrz\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba\",\n \"name\": \"d5d7fa0e-0178-458e-94e7-2895ae2998ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.349858893Z\",\n \"endTime\": \"2026-10-16T11:47:34.736973041Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba\n correlation ID: 2d09fb79-4f4d-4e95-9537-b8e2ea39de31\n client request ID: d79af74d-f260-41fd-865c-d23209b51fe0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f141cf40a40\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-s45bf5gctwrz\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba\",\n \"name\": \"d5d7fa0e-0178-458e-94e7-2895ae2998ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.349858893Z\",\n \"endTime\": \"2026-10-16T11:47:34.736973041Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d5d7fa0e-0178-458e-94e7-2895ae2998ba\n correlation ID: 2d09fb79-4f4d-4e95-9537-b8e2ea39de31\n client request ID: d79af74d-f260-41fd-865c-d23209b51fe0\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bf10f6f4044f1956f5889ded840cdbefc580d4fc82fa5c74dd1f0b76ca8d141f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"5cc11cee2be1a364a4a8f96d02040c08c6400f6da963a61909443a73c2e23f5a","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-v7kmcc with version 4.21.38\nUnexpected error:\n \u003c*fmt.wrapError | 0x210c156bc700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v7kmcc: failed waiting for cluster=\"np-version-upgrade-cluster-v7kmcc\" in resourcegroup=\"rg-np-version-upgrade-v7kmcc-xsth7mmz6mwq\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27\",\n \"name\": \"21480318-8ca1-438b-83d1-c0d4afadcb27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:29.86865973Z\",\n \"endTime\": \"2026-10-16T12:43:31.120266148Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27\n correlation ID: 88d3443b-33b0-4187-ae32-473011f1f8a9\n client request ID: a93ba081-83e5-408e-b984-1c88d2bcbd35\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-v7kmcc with version 4.21.38\nUnexpected error:\n \u003c*fmt.wrapError | 0x210c156bc700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v7kmcc: failed waiting for cluster=\"np-version-upgrade-cluster-v7kmcc\" in resourcegroup=\"rg-np-version-upgrade-v7kmcc-xsth7mmz6mwq\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27\",\n \"name\": \"21480318-8ca1-438b-83d1-c0d4afadcb27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:14:29.86865973Z\",\n \"endTime\": \"2026-10-16T12:43:31.120266148Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21480318-8ca1-438b-83d1-c0d4afadcb27\n correlation ID: 88d3443b-33b0-4187-ae32-473011f1f8a9\n client request ID: a93ba081-83e5-408e-b984-1c88d2bcbd35\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c317bae78d9b50542eff6cb1d2e74ca3fff1efaee294489061d64d6e5fd75ac3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"dcb01c2ddc87c79812f3a6b8a813ececccf2210f79f2960a1f189bc60733454e","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x16706e68afa0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-6dw2bx5ds9kb\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc\",\n \"name\": \"edc5b1cd-acae-47b3-aa88-4662fa83e6fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:21:30.614053227Z\",\n \"endTime\": \"2026-10-16T11:50:39.851463582Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc\n correlation ID: d0456136-5d4d-427b-b7a5-1c6f68dd0361\n client request ID: 83b4aaf0-43c3-43b5-a143-55b6ae1d78d1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0x16706e68afa0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-6dw2bx5ds9kb\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc\",\n \"name\": \"edc5b1cd-acae-47b3-aa88-4662fa83e6fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:21:30.614053227Z\",\n \"endTime\": \"2026-10-16T11:50:39.851463582Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edc5b1cd-acae-47b3-aa88-4662fa83e6fc\n correlation ID: d0456136-5d4d-427b-b7a5-1c6f68dd0361\n client request ID: 83b4aaf0-43c3-43b5-a143-55b6ae1d78d1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c8cf8203613a8e1625786aa476f877288808bec0f45e8a75402b771c46c3ba37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"d654a19b4359e6f92e0522049300de4d0adadb00699b4a6c37b2ea868ce4a692","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x345a0cb74ca0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-f5wd6886vcn2\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5\",\n \"name\": \"09427242-c235-4b25-9b05-5fc93af25ca5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:52.120671637Z\",\n \"endTime\": \"2026-10-16T11:47:59.844546185Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5\n correlation ID: be95e7ca-864f-4e1e-8ac2-79d1ca2c5a2a\n client request ID: 24b8da34-4220-4d6a-84a8-6aeb131431d9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x345a0cb74ca0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-f5wd6886vcn2\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5\",\n \"name\": \"09427242-c235-4b25-9b05-5fc93af25ca5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:52.120671637Z\",\n \"endTime\": \"2026-10-16T11:47:59.844546185Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09427242-c235-4b25-9b05-5fc93af25ca5\n correlation ID: be95e7ca-864f-4e1e-8ac2-79d1ca2c5a2a\n client request ID: 24b8da34-4220-4d6a-84a8-6aeb131431d9\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cc437196a2cad38f8c0a92dd222683fbfbc53320cba048a49d55257432a76754","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"a2f68ea0fe59a89d13063468f53fb9a82f7c10a74a1836c761d6416cb5beff47","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x29ceb663a360\u003e: \n failed to create HCP cluster ea-listjfw7qd: failed waiting for cluster=\"ea-listjfw7qd\" in resourcegroup=\"ea-list-dgf4pnnb4pjb\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8\",\n \"name\": \"4dac0821-0e5e-4879-8b02-6b86c3f9ddb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:51.217847726Z\",\n \"endTime\": \"2026-10-16T11:47:59.877781908Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8\n correlation ID: 47f07f5a-d8d8-453a-b9aa-aab49eab730a\n client request ID: 41282790-a233-46d8-b966-573e5286f1f4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0x29ceb663a360\u003e: \n failed to create HCP cluster ea-listjfw7qd: failed waiting for cluster=\"ea-listjfw7qd\" in resourcegroup=\"ea-list-dgf4pnnb4pjb\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8\",\n \"name\": \"4dac0821-0e5e-4879-8b02-6b86c3f9ddb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:51.217847726Z\",\n \"endTime\": \"2026-10-16T11:47:59.877781908Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4dac0821-0e5e-4879-8b02-6b86c3f9ddb8\n correlation ID: 47f07f5a-d8d8-453a-b9aa-aab49eab730a\n client request ID: 41282790-a233-46d8-b966-573e5286f1f4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cc4762f2113fb4d0d9d99db58e26b31c33bfcf980b46dfe002887ba720716c48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"3e127dc8a5033c99ca0089d8b9177daa434c14c29e269fe237c27a7ead0ffd82","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-rf4s5g\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a0d098322a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-rf4s5g: failed waiting for cluster=\"cp-ystream-upgrade-5-1-rf4s5g\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-zfstnv56jv54\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562\",\n \"name\": \"5b215067-e4fb-4ba6-a3c8-88327ce04562\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:44.075775976Z\",\n \"endTime\": \"2026-10-16T11:47:49.83736828Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562\n correlation ID: 1b44b5b8-3964-4666-b567-9729ea3b6cfb\n client request ID: 420bba0b-936e-4bc8-ab33-d28994da420f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-rf4s5g\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a0d098322a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-rf4s5g: failed waiting for cluster=\"cp-ystream-upgrade-5-1-rf4s5g\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-zfstnv56jv54\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562\",\n \"name\": \"5b215067-e4fb-4ba6-a3c8-88327ce04562\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:44.075775976Z\",\n \"endTime\": \"2026-10-16T11:47:49.83736828Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b215067-e4fb-4ba6-a3c8-88327ce04562\n correlation ID: 1b44b5b8-3964-4666-b567-9729ea3b6cfb\n client request ID: 420bba0b-936e-4bc8-ab33-d28994da420f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cd02cfcc15cba699ae699dfee1312b69b26e0ece1bafde29ea3246f78abc7bd8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"2d1cfe3f27ef0904cf407ecfb62faae7ea943492cd64b78f52ceb19f586165cf","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xe0368d38100\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-h4bzxzqvfdcw\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937\",\n \"name\": \"eb0f91c1-3b8b-4469-a615-126ec236e937\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:34.602862036Z\",\n \"endTime\": \"2026-10-16T11:47:35.486035854Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937\n correlation ID: 86eb3103-6942-44ec-a43b-2ba190a4ae32\n client request ID: 4d7beedb-0e11-4250-97e6-1589ca4e6668\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xe0368d38100\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-h4bzxzqvfdcw\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937\",\n \"name\": \"eb0f91c1-3b8b-4469-a615-126ec236e937\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:34.602862036Z\",\n \"endTime\": \"2026-10-16T11:47:35.486035854Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb0f91c1-3b8b-4469-a615-126ec236e937\n correlation ID: 86eb3103-6942-44ec-a43b-2ba190a4ae32\n client request ID: 4d7beedb-0e11-4250-97e6-1589ca4e6668\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d3babf0b92479b5424c4a059f66f29e3c21a5239790e580c6eb70c9083a4266f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"627de531eb0c6e8642332de2ab9eab9bc93f70b8522fb01255fbad68a41cf741","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x356517fb8480\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-gg2pb4grctp4\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5\",\n \"name\": \"cd8f2d5b-465a-406e-9cc2-e6c9587606f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:52.22518707Z\",\n \"endTime\": \"2026-10-16T11:47:59.816411083Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5\n correlation ID: 9cbbfb0c-2b84-41ef-8052-4fc6765ee876\n client request ID: 78dd1326-407d-4271-9447-caf037d2818a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x356517fb8480\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-gg2pb4grctp4\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5\",\n \"name\": \"cd8f2d5b-465a-406e-9cc2-e6c9587606f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:52.22518707Z\",\n \"endTime\": \"2026-10-16T11:47:59.816411083Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd8f2d5b-465a-406e-9cc2-e6c9587606f5\n correlation ID: 9cbbfb0c-2b84-41ef-8052-4fc6765ee876\n client request ID: 78dd1326-407d-4271-9447-caf037d2818a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d580072f1bd29f7a2ecff36bba516cc9c355f77e5944e3860c42d82b4cdbcac2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"3f03c97f235f01d6999956f4cc53d2f97cf7d8ec8ef75b33c128cf78a34ef8b5","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:421]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e5972a223c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-7j8469sk5tqr\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515\",\n \"name\": \"3838bb8e-dfbd-4e5a-9339-671595a03515\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.238054549Z\",\n \"endTime\": \"2026-10-16T11:47:34.729902173Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515\n correlation ID: 99bb95a5-8b87-4113-8926-d7de49dc11ce\n client request ID: 758d0f34-5d0d-498d-a7d0-60f7ca767673\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:421]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e5972a223c0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-7j8469sk5tqr\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515\",\n \"name\": \"3838bb8e-dfbd-4e5a-9339-671595a03515\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:25.238054549Z\",\n \"endTime\": \"2026-10-16T11:47:34.729902173Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3838bb8e-dfbd-4e5a-9339-671595a03515\n correlation ID: 99bb95a5-8b87-4113-8926-d7de49dc11ce\n client request ID: 758d0f34-5d0d-498d-a7d0-60f7ca767673\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d62899e9ae0d37559c11db53603276b1527b3bd0f4aa3ddc4b2791d628c3c5f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"e272ef11785652552361f893f5a45defb3d287faaaaf2b541ca001f902cbf560","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mggmm7\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xd7e57968900\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-mggmm7: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mggmm7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-xjmtnnp8qggs\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572\",\n \"name\": \"542436c6-ac27-49e3-a8f4-9fddb7746572\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:33.806164198Z\",\n \"endTime\": \"2026-10-16T11:47:34.778492912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572\n correlation ID: d197fcb4-7896-4d81-aa49-14527696e626\n client request ID: a66a415f-00c6-4bc2-907a-8ee32e11da3e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-mggmm7\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xd7e57968900\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-mggmm7: failed waiting for cluster=\"cp-ystream-upgrade-4-22-mggmm7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-xjmtnnp8qggs\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572\",\n \"name\": \"542436c6-ac27-49e3-a8f4-9fddb7746572\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:33.806164198Z\",\n \"endTime\": \"2026-10-16T11:47:34.778492912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/542436c6-ac27-49e3-a8f4-9fddb7746572\n correlation ID: d197fcb4-7896-4d81-aa49-14527696e626\n client request ID: a66a415f-00c6-4bc2-907a-8ee32e11da3e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d96b423b19fe2a52180cd3859a0f89d81e098cbc200426edf00c6683d8df2460","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"9647c888b9bb5db2ae4c02f3ef1f1edb946e9fd90922589e62703226b412cc0f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x249a5b185a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pkrtk8j7vt4h\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca\",\n \"name\": \"3ed5aa3d-aed2-41b9-8f51-718a07aca6ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:42.248685179Z\",\n \"endTime\": \"2026-10-16T11:47:49.853732372Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca\n correlation ID: fa48c745-f812-4375-9618-451ce1dbb645\n client request ID: 8468ac1c-82d8-4052-8947-0fbd7b155196\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x249a5b185a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pkrtk8j7vt4h\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca\",\n \"name\": \"3ed5aa3d-aed2-41b9-8f51-718a07aca6ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:42.248685179Z\",\n \"endTime\": \"2026-10-16T11:47:49.853732372Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ed5aa3d-aed2-41b9-8f51-718a07aca6ca\n correlation ID: fa48c745-f812-4375-9618-451ce1dbb645\n client request ID: 8468ac1c-82d8-4052-8947-0fbd7b155196\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d9b72edb60fd21d8cc99f65c6dadc2038080004ec4194059c7353f6f24d0bd36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"ae79e903fb8883e6a16fdc0341c2befe1b7b1220a63ea3653584d1992ada9cb3","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--sjp6pzqgf2j2/cluster-hs-pre-mr5ffk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x15cd9f62200\u003e: \n failed to create HCP cluster cluster-hs-pre-mr5ffk: failed waiting for cluster=\"cluster-hs-pre-mr5ffk\" in resourcegroup=\"rg-hs-pre--sjp6pzqgf2j2\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912\",\n \"name\": \"09d7932b-a3ed-490b-ad59-af083f46f912\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:45.646648105Z\",\n \"endTime\": \"2026-10-16T11:47:49.816471411Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912\n correlation ID: dad48d0a-2702-4ac2-bb83-5c8e5a1fcf05\n client request ID: 86ec96bd-5ab0-482b-bdd9-e1e77f04cdcf\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--sjp6pzqgf2j2/cluster-hs-pre-mr5ffk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x15cd9f62200\u003e: \n failed to create HCP cluster cluster-hs-pre-mr5ffk: failed waiting for cluster=\"cluster-hs-pre-mr5ffk\" in resourcegroup=\"rg-hs-pre--sjp6pzqgf2j2\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912\",\n \"name\": \"09d7932b-a3ed-490b-ad59-af083f46f912\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:45.646648105Z\",\n \"endTime\": \"2026-10-16T11:47:49.816471411Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/09d7932b-a3ed-490b-ad59-af083f46f912\n correlation ID: dad48d0a-2702-4ac2-bb83-5c8e5a1fcf05\n client request ID: 86ec96bd-5ab0-482b-bdd9-e1e77f04cdcf\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"df532d1cb40d231bbb63ee50c2475496517e59b3ab5604520666573b69110462","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"63b7a609dba699a3bb0a33d635a9e5513671bbfb16836cadb91c5d2dcd8c6f19","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-qr5nnd with version 4.21.38\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f1c2c46ab00\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qr5nnd: failed waiting for cluster=\"np-version-upgrade-cluster-qr5nnd\" in resourcegroup=\"rg-np-version-upgrade-qr5nnd-l8xjnzcdr6f9\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce\",\n \"name\": \"31daeda6-8efc-4902-9484-e978fd605dce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:12:31.330566092Z\",\n \"endTime\": \"2026-10-16T12:41:31.112317819Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce\n correlation ID: cf017195-c27a-482e-a585-164ee741970d\n client request ID: 2b1b472c-af6e-4011-b54c-755047fe28c2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-qr5nnd with version 4.21.38\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f1c2c46ab00\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qr5nnd: failed waiting for cluster=\"np-version-upgrade-cluster-qr5nnd\" in resourcegroup=\"rg-np-version-upgrade-qr5nnd-l8xjnzcdr6f9\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce\",\n \"name\": \"31daeda6-8efc-4902-9484-e978fd605dce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T12:12:31.330566092Z\",\n \"endTime\": \"2026-10-16T12:41:31.112317819Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-06/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/31daeda6-8efc-4902-9484-e978fd605dce\n correlation ID: cf017195-c27a-482e-a585-164ee741970d\n client request ID: 2b1b472c-af6e-4011-b54c-755047fe28c2\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e28bbab28f0b69a0f78a91f9a47e40bd32847933f97cfb451e4652465689ca56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"0accb990492557d084ad00c9a810ebe72dd8287b7bc8d9d394a79d3ec69fbd86","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:96]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3edf0bbb68a0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-z529cprjbb9b\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7\",\n \"name\": \"eff59990-4aa9-4e5e-87d9-e72bb79446b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:46.389668007Z\",\n \"endTime\": \"2026-10-16T11:47:49.83909108Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7\n correlation ID: 8637dbbe-5f4a-484b-9a7f-0752af5bfdf2\n client request ID: 834ff1df-c90f-4eb9-9740-eca7c029338e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:96]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0x3edf0bbb68a0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-z529cprjbb9b\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7\",\n \"name\": \"eff59990-4aa9-4e5e-87d9-e72bb79446b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:46.389668007Z\",\n \"endTime\": \"2026-10-16T11:47:49.83909108Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eff59990-4aa9-4e5e-87d9-e72bb79446b7\n correlation ID: 8637dbbe-5f4a-484b-9a7f-0752af5bfdf2\n client request ID: 834ff1df-c90f-4eb9-9740-eca7c029338e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e8071a4b7ec89d9540a26189e6b798dfb119924f967e22c644ffd2e7ec450c44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"c4d0ed7d70c4e2723c49ed37fb9159b5ca2ef302da886789f78b6a44414ea242","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-vxglcs\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c50ef51c3c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-vxglcs: failed waiting for cluster=\"cp-ystream-upgrade-4-21-vxglcs\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-v7bk8rsxngbc\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2\",\n \"name\": \"fc0e3d77-531c-4ab5-be8c-bb319d76c5a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:43.252078766Z\",\n \"endTime\": \"2026-10-16T11:47:49.83582259Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2\n correlation ID: 273c409a-34e8-4c3e-83cc-b70094e63494\n client request ID: c6c50f5f-b0c7-40d0-a975-1c325d678573\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-vxglcs\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c50ef51c3c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-vxglcs: failed waiting for cluster=\"cp-ystream-upgrade-4-21-vxglcs\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-v7bk8rsxngbc\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2\",\n \"name\": \"fc0e3d77-531c-4ab5-be8c-bb319d76c5a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:43.252078766Z\",\n \"endTime\": \"2026-10-16T11:47:49.83582259Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc0e3d77-531c-4ab5-be8c-bb319d76c5a2\n correlation ID: 273c409a-34e8-4c3e-83cc-b70094e63494\n client request ID: c6c50f5f-b0c7-40d0-a975-1c325d678573\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ec337b3e924b26d5a2f257aab29096402e1d12944c4e213a9020c102e18d4280","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"b93bac5ee55d46ed1d89154858d9b08bd23b63b8774e6e33f1c1c14ab66fca7f","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x248dbdd6a2d0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-p9pjc5tph49f\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-p9pjc5tph49f\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x248dbde747e0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x248dbdd6a2d0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-p9pjc5tph49f\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-p9pjc5tph49f\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x248dbde747e0\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-p9pjc5tph49f\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-p9pjc5tph49f, error: context deadline exceeded"}},{"row_id":"ee3b69b5cda3d7328993dba90f943bd231beb82f12fb167d5441e883905ffe3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"4be11067a4063ad348db95dbcf80fdc2805fe4486a6b4ce644af9294d77bbbb5","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a684268b1e0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-h7n8bhdfxkhz\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff\",\n \"name\": \"24d756d8-8406-403a-803e-d72f60212fff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:14.969975943Z\",\n \"endTime\": \"2026-10-16T11:47:15.076712864Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff\n correlation ID: 830ad185-3324-4619-a20a-fcad371ddf91\n client request ID: 4185ab5a-305d-4830-a594-26d6ec9cbb8f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a684268b1e0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001-h7n8bhdfxkhz\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff\",\n \"name\": \"24d756d8-8406-403a-803e-d72f60212fff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:14.969975943Z\",\n \"endTime\": \"2026-10-16T11:47:15.076712864Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24d756d8-8406-403a-803e-d72f60212fff\n correlation ID: 830ad185-3324-4619-a20a-fcad371ddf91\n client request ID: 4185ab5a-305d-4830-a594-26d6ec9cbb8f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f1ff2154ef41dc157189a3f2cafbe18edf90ba39fbd9350d9ce356dd7942500d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"9abb8da4d45c27b5c241d75cfb1dcf171e174ac6a95fe33a4e43b4d92dc7e7bf","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x2fb9619ac660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mqjl28m5r6sd\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454\",\n \"name\": \"9a2fd85c-d6eb-404b-bbec-4b9f90f3d454\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:19:55.333209171Z\",\n \"endTime\": \"2026-10-16T11:48:59.82480605Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454\n correlation ID: f229ed44-de68-4ac8-91bd-519680e0c83b\n client request ID: 6dc6aa5e-bf11-4175-a256-7174e23bec50\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x2fb9619ac660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mqjl28m5r6sd\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454\",\n \"name\": \"9a2fd85c-d6eb-404b-bbec-4b9f90f3d454\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:19:55.333209171Z\",\n \"endTime\": \"2026-10-16T11:48:59.82480605Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a2fd85c-d6eb-404b-bbec-4b9f90f3d454\n correlation ID: f229ed44-de68-4ac8-91bd-519680e0c83b\n client request ID: 6dc6aa5e-bf11-4175-a256-7174e23bec50\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f87cfbbb4e76d3cd10ea735cd7e934d82e1a09a6e102ffdfc5d8f2182836f13d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"alert","signature_id":"3f4e95f4a56ef2812656081fda4421f50ba31daadd1af199afbcdf17ce2e3431","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-16T11:54:17Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j9203200-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9203200/providers/microsoft.monitor/accounts/services-j9203200\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of completed cluster create (install) operations on ci01-j9203200-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired"}},{"row_id":"fa7af82d8f296041ed10a0b1234a02da2441e17ceac7d887825fbd1ceb179359","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"d174aa6363d992122c49f4fc21a9483030773776409c9e301f2a19c69d947b2a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xde395ae420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-sc7wtvpvc55m\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507\",\n \"name\": \"1512da1f-5680-47e6-aaa6-c5133ea9f507\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:45.126771542Z\",\n \"endTime\": \"2026-10-16T11:47:49.832518532Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507\n correlation ID: aec181d7-e9c4-4a80-b7f9-caa74be2736d\n client request ID: 7881578f-b798-428c-aa0c-b9a97e3da350\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xde395ae420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-sc7wtvpvc55m\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507\",\n \"name\": \"1512da1f-5680-47e6-aaa6-c5133ea9f507\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:18:45.126771542Z\",\n \"endTime\": \"2026-10-16T11:47:49.832518532Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1512da1f-5680-47e6-aaa6-c5133ea9f507\n correlation ID: aec181d7-e9c4-4a80-b7f9-caa74be2736d\n client request ID: 7881578f-b798-428c-aa0c-b9a97e3da350\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fbf3d84baea5c00320374a08ce5de3b79958e25f6739c677de6177d7c4e4a471","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"99c69765f43fb35fd9ac8d86d7dd4823005533fe8821d9261a6c6ec54ba28e58","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ddbb8172580\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gghkqpprl5hx\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5\",\n \"name\": \"0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:20:15.848246532Z\",\n \"endTime\": \"2026-10-16T11:49:19.883725425Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5\n correlation ID: 6a9850a7-3bdb-423f-983c-0f81dde37610\n client request ID: 285c03b5-ab82-40d1-bf50-841a421f3e6b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ddbb8172580\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gghkqpprl5hx\" to finish creating: GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5\",\n \"name\": \"0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:20:15.848246532Z\",\n \"endTime\": \"2026-10-16T11:49:19.883725425Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ffb0529-4ef5-418f-ac95-b1e9e08ba9f5\n correlation ID: 6a9850a7-3bdb-423f-983c-0f81dde37610\n client request ID: 285c03b5-ab82-40d1-bf50-841a421f3e6b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fd087fccec2becaf0fe36adf1ec6877e1ed21057948154c54d968a50f1e5e86c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111041799249203200","occurred_at":"2026-10-16T10:29:17Z","failed_at":"e2e","signature_id":"cd976b5f6960ce8d52b4b03b10926d1a311c4c83fb44987456f12b9d1d004aac","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x3e31c88336b0\u003e: \n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802\",\n \"name\": \"dd66ee81-312a-43d3-9e03-4a33287de802\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:20:24.302026234Z\",\n \"endTime\": \"2026-10-16T11:49:29.826835151Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802\n correlation ID: eedbd3ff-6629-45c2-8504-b9a36ee198f7\n client request ID: df570f53-90be-4439-a189-428c0f483b8a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x3e31c88336b0\u003e: \n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802\",\n \"name\": \"dd66ee81-312a-43d3-9e03-4a33287de802\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:20:24.302026234Z\",\n \"endTime\": \"2026-10-16T11:49:29.826835151Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-02-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9203200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dd66ee81-312a-43d3-9e03-4a33287de802\n correlation ID: eedbd3ff-6629-45c2-8504-b9a36ee198f7\n client request ID: df570f53-90be-4439-a189-428c0f483b8a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [dataPlaneOIDCFederation] data-plane OIDC federation not yet ensured for 3 operator(s): disk-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-disk-csi-driver), file-csi-driver (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver), image-registry (/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-image-registry); [hypershiftHostedCluster] hosted cluster state not cached yet; [roleAssignments] role assignments not yet confirmed; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":72,"failure_pattern_ids":["0cd69af8904472f73d329003c8c5db9d0e892958b3af0d73d1a1dee951ccff6d","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","6685a5a6bc3f263f200f46fbec1e8851219602ada5b75c61eac4345b058adb70","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","8aeee60fde47313471f5f7079969fc517be093d137df1279e92b7532f0d725de","96d801deab41ec98748c7082577ee746091d92728a5e786bd4eef6a4d1b2ca0f","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","9e7d58cf80cefdbf58d438cd330cbadc1e097f5d57f87fa6133331546f9b5534","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":72,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7576/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111038960766554112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7576,"pr_state":"open","pr_sha":"fa269861a2b22a2b36dce3160f9f49c19f7e1350","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T10:18:04Z","started_at":"2026-10-16T10:18:04Z","completed_at":"2026-10-16T10:44:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T10:58:16.901423842Z","timing_metadata_checked_at":"2026-10-16T10:58:16.901423842Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T10:58:16.905090629Z","region_metadata_checked_at":"2026-10-16T10:58:16.905090629Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"95d3734dcf4b4b26974deddaffb31912ff8a9ea801cfbb006f176dbe1d215e6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7576/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111038960766554112","occurred_at":"2026-10-16T10:18:04Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T10:37:16.131Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-16T10:37:17.576Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-16T10:37:18.879Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6554112-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d\ntime=2026-10-16T10:39:19.372Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.Resources/deployments/182dddaef1eb2eeeaeb1c74f26cc60d61fef94e53fb090c30b83b72986af948d/operationStatuses/08584094598472503244\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.Resources/deployments/cluster-ismqifijm7koe\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6554112-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j6554112-svc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j6554112-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111034368125308928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7387,"pr_state":"closed","pr_sha":"14770cd53f0f4c113c23a8835a0dc3e9f4e40aeb","final_merged_sha":"14770cd53f0f4c113c23a8835a0dc3e9f4e40aeb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T09:59:49Z","started_at":"2026-10-16T09:59:49Z","completed_at":"2026-10-16T12:25:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T12:43:22.005044919Z","timing_metadata_checked_at":"2026-10-16T12:43:22.005044919Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T12:43:22.009021107Z","region_metadata_checked_at":"2026-10-16T12:43:22.009021107Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6e75c08e0dfecc130e5689f41a41915f0ab4766574a95aa0be389eedc37af9a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111034368125308928","occurred_at":"2026-10-16T09:59:49Z","failed_at":"alert","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T11:18:08Z\n Ended: 2026-10-16T11:34:08Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5308928-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T11:18:08Z\n Ended: 2026-10-16T11:34:07Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5308928-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111032815893090304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7292,"pr_state":"closed","pr_sha":"e2249e5ba4dbcc59216d7341da5783394710eb68","final_merged_sha":"e2249e5ba4dbcc59216d7341da5783394710eb68","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T09:53:39Z","started_at":"2026-10-16T09:53:39Z","completed_at":"2026-10-16T12:12:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T12:22:36.169028663Z","timing_metadata_checked_at":"2026-10-16T12:22:36.169028663Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T12:22:36.192238891Z","region_metadata_checked_at":"2026-10-16T12:22:36.192238891Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d79aab1fe585df5d264fcc92e1a476ac68db5b06db6e50f5f320e9abefa8f7e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111032815893090304","occurred_at":"2026-10-16T09:53:39Z","failed_at":"e2e","signature_id":"0954bd1b1bf2ab0efaa5338771ec18987438624760aa570ab6359e164921f5fe","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3d49e0c9b600\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cqn2ccl8wpmh\" to finish creating: GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\",\n \"name\": \"57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:01:48.232534133Z\",\n \"endTime\": \"2026-10-16T11:20:52.453260342Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NodeDoesNotExist: Node does not exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NotReady: * Deleting: Machine deletion in progress, stage: WaitingForInfrastructureDeletion\\n* HealthCheckSucceeded: Health check failed: Node failed to report startup in 10m0s\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\n correlation ID: f387d1ed-05da-402f-8ada-b632f0d02aae\n client request ID: 36e27b8d-f92f-4579-9e74-48bd79aecb10\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3d49e0c9b600\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cqn2ccl8wpmh\" to finish creating: GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\",\n \"name\": \"57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T11:01:48.232534133Z\",\n \"endTime\": \"2026-10-16T11:20:52.453260342Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NodeDoesNotExist: Node does not exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0j2o7l0q7m7m7f-np-delete-main-4ql8g-j89nf: NotReady: * Deleting: Machine deletion in progress, stage: WaitingForInfrastructureDeletion\\n* HealthCheckSucceeded: Health check failed: Node failed to report startup in 10m0s\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3090304.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57ef0bb1-ee3a-46b4-87ee-9e92b66bb5c8\n correlation ID: f387d1ed-05da-402f-8ada-b632f0d02aae\n client request ID: 36e27b8d-f92f-4579-9e74-48bd79aecb10","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-lsp7k9q4q2pz\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111032217462378496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7575,"pr_state":"open","pr_sha":"2de625874e11113b57e1e9685f1a4f1c0bd92481","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T09:51:15Z","started_at":"2026-10-16T09:51:15Z","completed_at":"2026-10-16T12:11:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T12:22:36.605735604Z","timing_metadata_checked_at":"2026-10-16T12:22:36.605735604Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T12:22:36.609078094Z","region_metadata_checked_at":"2026-10-16T12:22:36.609078094Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b4462056b0a8d1bcd52bf9cc4bc3d4c9aa7da0e5dc0926943cd6f1da344123dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111032217462378496","occurred_at":"2026-10-16T09:51:15Z","failed_at":"e2e","signature_id":"f7c1c17ba2f883d4fd6f04f522549121b520b9cd1aa76eeb64893db3518e6496","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c2c57986ce0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1c2c57986cc0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1c2c56cc4500\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1c2c57986ca0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1c2c596cc000\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c2c57986ce0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1c2c57986cc0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1c2c56cc4500\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1c2c57986ca0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gbltgmkqfq7l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T10:45:41.1909764Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1c2c596cc000\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111028857451581440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7502,"pr_state":"closed","pr_sha":"4e388171c7cac99a0d73a3ae18b94e46f3e3866f","final_merged_sha":"4e388171c7cac99a0d73a3ae18b94e46f3e3866f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T09:37:55Z","started_at":"2026-10-16T09:37:55Z","completed_at":"2026-10-16T11:44:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T12:00:57.143823469Z","timing_metadata_checked_at":"2026-10-16T12:00:57.143823469Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T12:00:57.147320658Z","region_metadata_checked_at":"2026-10-16T12:00:57.147320658Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111024523175268352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7574,"pr_state":"closed","pr_sha":"4010c817ad8822057dfe4c09c41154d9cfeb26fb","final_merged_sha":"fd0887d3395f226558878e178d6c4476bbb74aec","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T09:20:41Z","started_at":"2026-10-16T09:20:41Z","completed_at":"2026-10-16T11:25:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T11:40:15.996693003Z","timing_metadata_checked_at":"2026-10-16T11:40:15.996693003Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T11:40:16.001968986Z","region_metadata_checked_at":"2026-10-16T11:40:16.001968986Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6bd9a1ce448f8939358fad42d2fa8745ef5d61c8da0cdbb5da82fe4a53b146c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111024523175268352","occurred_at":"2026-10-16T09:20:41Z","failed_at":"e2e","signature_id":"884768de414f2c6367d7336de5c7997f81daf783ffd0d385d67afc2eda8eab1a","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33cb56e17020\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4xcd2z54mnjl\" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\",\n \"name\": \"ed1bfb49-b931-4647-a8ac-58b583c3b50b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T10:16:52.135966436Z\",\n \"endTime\": \"2026-10-16T10:35:53.058795763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-pw27v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4: Failed: virtualmachine failed to create or update. err: failed to get existing resource private-keyvault-4xcd2z54mnjl--managed/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xcd2z54mnjl--managed/providers/Microsoft.Compute/virtualMachines/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\n correlation ID: ef5c7325-64f6-4ca1-87ca-c2c3e96bf69a\n client request ID: 392329dd-b2a3-4225-a4e8-154f2f223edb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33cb56e17020\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4xcd2z54mnjl\" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\",\n \"name\": \"ed1bfb49-b931-4647-a8ac-58b583c3b50b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T10:16:52.135966436Z\",\n \"endTime\": \"2026-10-16T10:35:53.058795763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-pw27v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4: Failed: virtualmachine failed to create or update. err: failed to get existing resource private-keyvault-4xcd2z54mnjl--managed/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xcd2z54mnjl--managed/providers/Microsoft.Compute/virtualMachines/s2n7n1o2s1t5d5z-np-1-m5qjf-49zn4\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3871b527-fb1e-4123-b38b-3cb2445a9fc8' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed1bfb49-b931-4647-a8ac-58b583c3b50b\n correlation ID: ef5c7325-64f6-4ca1-87ca-c2c3e96bf69a\n client request ID: 392329dd-b2a3-4225-a4e8-154f2f223edb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b7a7316d7bc9a13546759ba543c0393779c1adcbaf7eafe296163143bdb55b2c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b8557db0e923925db0b23e889ae37a3cf5a735e230e5c6b7a605971251049363","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7574/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111024523175268352","occurred_at":"2026-10-16T09:20:41Z","failed_at":"e2e","signature_id":"a18b5cfc4a13cbde1b572bc91b24cdf754420812d9b921eac4fc1bfe1226db6a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1eddd58959c0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-brlqmmsvjbp7\" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\",\n \"name\": \"1ec84baa-1f08-44e4-8884-c7d91686d7f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T10:26:32.519187505Z\",\n \"endTime\": \"2026-10-16T10:45:34.937263417Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\n correlation ID: 6fd1b613-3112-4dbd-9dff-6f44c7554f29\n client request ID: 0ac1b9cb-8173-4c03-9f5c-ab7b1db7762b\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1eddd58959c0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-brlqmmsvjbp7\" to finish creating: GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\",\n \"name\": \"1ec84baa-1f08-44e4-8884-c7d91686d7f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T10:26:32.519187505Z\",\n \"endTime\": \"2026-10-16T10:45:34.937263417Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-brlqmmsvjbp7--managed/providers/Microsoft.Compute/virtualMachines/f1s8b5f8y5z5b5s-autoscale-noaz-4zlkx-pm4sh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5268352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1ec84baa-1f08-44e4-8884-c7d91686d7f5\n correlation ID: 6fd1b613-3112-4dbd-9dff-6f44c7554f29\n client request ID: 0ac1b9cb-8173-4c03-9f5c-ab7b1db7762b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b7a7316d7bc9a13546759ba543c0393779c1adcbaf7eafe296163143bdb55b2c","d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111022552083075072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7483,"pr_state":"open","pr_sha":"b0495e84f608044ba73de9aa16c14970fb096b1f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T09:12:51Z","started_at":"2026-10-16T09:12:51Z","completed_at":"2026-10-16T11:24:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T11:40:15.613124357Z","timing_metadata_checked_at":"2026-10-16T11:40:15.613124357Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T11:40:15.616698046Z","region_metadata_checked_at":"2026-10-16T11:40:15.616698046Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b8b41b99d753feb636943e8f052a8ec0de38f8487cf3bb773ab49f7122dd6cff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111022552083075072","occurred_at":"2026-10-16T09:12:51Z","failed_at":"e2e","signature_id":"ed2036aae2cf00dc5018c7a58da50f85fd92d64f0bb21b758adcb9a18d2da21c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5mfh5m\" for cluster \"cluster-candidate-4-23-5mfh5m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x83c94b64840\u003e: \n failed to create NodePool np-5mfh5m: failed waiting for nodepool=\"np-5mfh5m\" for cluster \"cluster-candidate-4-23-5mfh5m\" in resourcegroup=\"rg-candidate-4-23-t8lk4npslqvb\" to finish creating: GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\",\n \"name\": \"4ba6beec-b4a8-4b19-9703-a8d4018563a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T10:16:52.661682642Z\",\n \"endTime\": \"2026-10-16T10:35:56.734294404Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\n correlation ID: 2de83bb6-2187-443d-9a87-45e45c14da85\n client request ID: 229ecc64-817b-4213-bc11-c707f778c1d6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5mfh5m\" for cluster \"cluster-candidate-4-23-5mfh5m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x83c94b64840\u003e: \n failed to create NodePool np-5mfh5m: failed waiting for nodepool=\"np-5mfh5m\" for cluster \"cluster-candidate-4-23-5mfh5m\" in resourcegroup=\"rg-candidate-4-23-t8lk4npslqvb\" to finish creating: GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\",\n \"name\": \"4ba6beec-b4a8-4b19-9703-a8d4018563a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T10:16:52.661682642Z\",\n \"endTime\": \"2026-10-16T10:35:56.734294404Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-t8lk4npslqvb-candidate-5mfh5m--managed/providers/Microsoft.Compute/virtualMachines/w2t3o1i6q4w1u2r-np-5mfh5m-477cf-hfl4n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3075072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4ba6beec-b4a8-4b19-9703-a8d4018563a9\n correlation ID: 2de83bb6-2187-443d-9a87-45e45c14da85\n client request ID: 229ecc64-817b-4213-bc11-c707f778c1d6\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111016450129924096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"closed","pr_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","final_merged_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T08:48:34Z","started_at":"2026-10-16T08:48:34Z","completed_at":"2026-10-16T10:59:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T11:08:22.49012708Z","timing_metadata_checked_at":"2026-10-16T11:08:22.49012708Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T11:08:22.495315263Z","region_metadata_checked_at":"2026-10-16T11:08:22.495315263Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7559/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111011832884891648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7559,"pr_state":"open","pr_sha":"6638a4e99213612f403fffdc5421fdd9c4655048","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T08:30:16Z","started_at":"2026-10-16T08:30:16Z","completed_at":"2026-10-16T09:20:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T09:33:37.582274666Z","timing_metadata_checked_at":"2026-10-16T09:33:37.582274666Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T09:33:37.58719395Z","region_metadata_checked_at":"2026-10-16T09:33:37.58719395Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"04684ca6012bb0a4a2944a519607a794eb36814d957aca850ad7f88bc01f5427","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7559/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111011832884891648","occurred_at":"2026-10-16T08:30:16Z","failed_at":"provision","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T09:14:54.862Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-16T09:15:30.118Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d to arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d.\\nThe image will still be available under it's original digest sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d in the target registry.\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d --from-registry-config /tmp/tmp.jIM7uBoqB8/containers/auth.json --to-registry-config /tmp/tmp.jIM7uBoqB8/containers/auth.json\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a","failure_pattern":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/nshneor/cs:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json","search_query":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d arohcpsvcdev.azurecr.io/nshneor/cs:7a9bf9af09700e173a8b419b48d774a418032d526bf027ac86490be261e6279d --from-registry-config /tmp/tmp.jIM7uBoqB8/containers/auth.json --to-registry-config /tmp/tmp.jIM7uBoqB8/containers/auth.json"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110999442776657920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7566,"pr_state":"closed","pr_sha":"659b8ce018841b2a037ed9a100f8353dfc9e1551","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T07:41:02Z","started_at":"2026-10-16T07:41:02Z","completed_at":"2026-10-16T10:19:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T10:37:05.624816891Z","timing_metadata_checked_at":"2026-10-16T10:37:05.624816891Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T10:37:05.629142877Z","region_metadata_checked_at":"2026-10-16T10:37:05.629142877Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"491c9ee6cfa9f313ed100171871746c7d69ee1c83bfac2bb7942f91a9fec7a8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110999442776657920","occurred_at":"2026-10-16T07:41:02Z","failed_at":"e2e","signature_id":"4e0ccb0d44ba37e343acca65141914d09bdba66d4aea11c0a4acc789d41b1f7b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3408412f86c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-nvxdbn5qmgtb\" to finish creating: GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\",\n \"name\": \"a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T08:39:54.399138921Z\",\n \"endTime\": \"2026-10-16T08:58:56.325845484Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\n correlation ID: 956ae344-8430-4fa1-b7b7-d3523f2ec507\n client request ID: 149dc074-a954-4585-b9e7-50faa51cea7a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3408412f86c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-nvxdbn5qmgtb\" to finish creating: GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\",\n \"name\": \"a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T08:39:54.399138921Z\",\n \"endTime\": \"2026-10-16T08:58:56.325845484Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-nvxdbn5qmgtb--managed/providers/Microsoft.Compute/virtualMachines/p8r1u9p3j7r2d5c-np-1-gkb8v-jqcmv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6657920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a18fbb87-8bb7-4a74-8f21-e96af9bfcf16\n correlation ID: 956ae344-8430-4fa1-b7b7-d3523f2ec507\n client request ID: 149dc074-a954-4585-b9e7-50faa51cea7a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7abd8d2ac3450c2bc85d321ce23fb7f0a267665c4ad1c2cd723a08de916101b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110999442776657920","occurred_at":"2026-10-16T07:41:02Z","failed_at":"e2e","signature_id":"bc274b8e050d4e114121e021987f8aef108de68d75ca046fc0ac61305af93eda","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x21a95a4b0000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x21a95b98e500\u003e{\n s: \"dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x21a95a4b0000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-4hk6sg2whd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x21a95b98e500\u003e{\n s: \"dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-2tfp6vu1t0esj9s2f87m52kogvfeb12i-d8h3a4b2v3x4x1h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"01a075ff256ceb0e0b1d938a88d093e2a699f17fc729f7caecc4c334022cc991","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j6657920-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["01a075ff256ceb0e0b1d938a88d093e2a699f17fc729f7caecc4c334022cc991","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110993516539678720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"closed","pr_sha":"81f928f6f3ca7595cc9207b59cd4cbb9fefc18d0","final_merged_sha":"09587f6f09c91b53ea705711a675d523def6cece","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T07:17:29Z","started_at":"2026-10-16T07:17:29Z","completed_at":"2026-10-16T09:37:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T09:55:11.333573296Z","timing_metadata_checked_at":"2026-10-16T09:55:11.333573296Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T09:55:11.337463883Z","region_metadata_checked_at":"2026-10-16T09:55:11.337463883Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"cd8560efab8ac3ec19aa8b7ae68ec27f087484579cfdbe47f8c9ffc1f24ed7a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110993516539678720","occurred_at":"2026-10-16T07:17:29Z","failed_at":"e2e","signature_id":"0f446af22185bf42597db517e6b3a284b2a5b5db844a92845ed8be29a77818cb","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x11ea08e84580\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x11ea09d18c20\u003e{\n s: \"dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x11ea08e84580\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-lc82txzvgr45/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x11ea09d18c20\u003e{\n s: \"dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-2tfouap5pj715l32pn7smi3ihfg5rglm-y5e3f0u2q1z1x6w: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa2af3baaf9b229c16ea2eedc12bed8c9ba9d5ba824e8a6c12d3ccdfa46d6b6d","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: dev/centralus/ci01-j9678720-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}},{"row_id":"e0ed217a86ec786184eac76d0f9fabff81756d4027c224df2df37380179543e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110993516539678720","occurred_at":"2026-10-16T07:17:29Z","failed_at":"e2e","signature_id":"67d8158e7a98b23599a95a7df825525e560fd538e129d6bd8ff14bb6c6993c02","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x8a369199c00\u003e: \n DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x8a36a1383e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x8a369199c00\u003e: \n DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x8a36a1383e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2qggj.apps.aro.v20260901.29of.j9678720.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","fa2af3baaf9b229c16ea2eedc12bed8c9ba9d5ba824e8a6c12d3ccdfa46d6b6d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110987944524255232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"closed","pr_sha":"661e26501ed1114621742809ef3d8488c795cc60","final_merged_sha":"8849e5d7c268e9779b73b664bf5de466df0d0ef6","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T06:55:20Z","started_at":"2026-10-16T06:55:20Z","completed_at":"2026-10-16T09:11:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T09:23:03.350902629Z","timing_metadata_checked_at":"2026-10-16T09:23:03.350902629Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T09:23:03.354395917Z","region_metadata_checked_at":"2026-10-16T09:23:03.354395917Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110961362036330496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"closed","pr_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","final_merged_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T05:09:43Z","started_at":"2026-10-16T05:09:43Z","completed_at":"2026-10-16T07:31:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T07:49:24.537052143Z","timing_metadata_checked_at":"2026-10-16T07:49:24.537052143Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T07:49:24.540690231Z","region_metadata_checked_at":"2026-10-16T07:49:24.540690231Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4dc41a1399992675e0767bc40ba7e5ec90392ec022bd5e53a7cedc0c136924e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110961362036330496","occurred_at":"2026-10-16T05:09:43Z","failed_at":"alert","signature_id":"78f5fc43246695b2dd39eca1a26a0ab5a62d3c615d9af627f94a79b10d68dc72","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s) (18 unknown, 2 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T06:22:59Z\n Ended: 2026-10-16T06:43:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s\", pod=\"azure-disk-csi-driver-controller-57b68cc787-snbg5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s/azure-disk-csi-driver-controller-57b68cc787-snbg5 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T06:22:59Z\n Ended: 2026-10-16T06:43:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t\", pod=\"ignition-server-proxy-7bf77dbb94-nn9mf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-nn9mf has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:40:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i\", pod=\"ignition-server-proxy-5ff57d4f88-chbqq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-chbqq has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s\", pod=\"azure-disk-csi-driver-controller-57b68cc787-mhkrv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s/azure-disk-csi-driver-controller-57b68cc787-mhkrv has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:40:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i\", pod=\"ignition-server-proxy-5ff57d4f88-xdrhg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-xdrhg has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t\", pod=\"ignition-server-proxy-7bf77dbb94-qgvpq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-qgvpq has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t\", pod=\"kube-controller-manager-d9d7765bd-fkcfl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/kube-controller-manager-d9d7765bd-fkcfl has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t\", pod=\"kube-controller-manager-d9d7765bd-4mm6q\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/kube-controller-manager-d9d7765bd-4mm6q has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t\", pod=\"ignition-server-proxy-7bf77dbb94-5mxk8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-5mxk8 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t\", pod=\"ignition-server-proxy-7bf77dbb94-tst84\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-tst84 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:40:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i\", pod=\"ignition-server-proxy-5ff57d4f88-qd9pm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-qd9pm has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:40:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i\", pod=\"ignition-server-proxy-5ff57d4f88-lwxgz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-lwxgz has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:40:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i\", pod=\"ignition-server-proxy-5ff57d4f88-md46c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-md46c has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t\", pod=\"ignition-server-proxy-7bf77dbb94-llgzf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/ignition-server-proxy-7bf77dbb94-llgzf has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s\", pod=\"azure-disk-csi-driver-controller-794b54859-f2fnv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0vohso5t63liv55elvvbd2lcm5o4-r0u1i3i9s5c3p5s/azure-disk-csi-driver-controller-794b54859-f2fnv has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:40:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i\", pod=\"ignition-server-proxy-5ff57d4f88-2clfg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-2clfg has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:40:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i\", pod=\"ignition-server-proxy-5ff57d4f88-sb22k\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn0udd3el5q8k9a92sklo58np0ejd1-t5k1k4l5b0o9q2i/ignition-server-proxy-5ff57d4f88-sb22k has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-16T06:23:00Z\n Ended: 2026-10-16T06:43:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6330496-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t\", pod=\"kube-controller-manager-d9d7765bd-nw48g\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfn1v5k3eq6o8cu3f0p12ngn3dgo1gm-g2c9w4j2t7t9v8t/kube-controller-manager-d9d7765bd-nw48g has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110960931310669824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7564,"pr_state":"closed","pr_sha":"fe73be4e2ed407567b671158287fa9c3aa4a6d6b","final_merged_sha":"fe73be4e2ed407567b671158287fa9c3aa4a6d6b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T05:08:00Z","started_at":"2026-10-16T05:08:00Z","completed_at":"2026-10-16T07:18:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T07:28:55.579179986Z","timing_metadata_checked_at":"2026-10-16T07:28:55.579179986Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T07:28:55.584874568Z","region_metadata_checked_at":"2026-10-16T07:28:55.584874568Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6d4df8c0640a678af73bad45e4ce4e15237d8d4364b66d43a7e12e73b1dafaa9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110960931310669824","occurred_at":"2026-10-16T05:08:00Z","failed_at":"e2e","signature_id":"f816a0ce46c83f26ec513b8e3bd7f4adea18fdccd58c65277a306190fa1fe08f","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x202498ad04a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x202498ad0480\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x202497dfe410\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x202498ad0460\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x202499acacc0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x202498ad04a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x202498ad0480\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x202497dfe410\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x202498ad0460\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2mg24zlthd9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T05:58:01.2930251Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x202499acacc0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}},{"row_id":"b3a2f95c591af5604bd70adff18bcd2dc44937f8f58483c403a1d784105d08c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110960931310669824","occurred_at":"2026-10-16T05:08:00Z","failed_at":"e2e","signature_id":"273e478ae270b5bcb7f9c0960c08545dbd11753951d5c582aa22856eef08182c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x1583fe428d80\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qxkjhfl8b89k\" to finish creating: GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\",\n \"name\": \"fb248cdb-708c-4e1a-a191-a9d66edb0204\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T06:10:02.689570443Z\",\n \"endTime\": \"2026-10-16T06:29:04.095727153Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\n correlation ID: 22bcc657-19c6-431f-9432-afe9f54b1a18\n client request ID: 0f3d0f1f-4c8a-4108-a1df-7f5cda2678a1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x1583fe428d80\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qxkjhfl8b89k\" to finish creating: GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\",\n \"name\": \"fb248cdb-708c-4e1a-a191-a9d66edb0204\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-16T06:10:02.689570443Z\",\n \"endTime\": \"2026-10-16T06:29:04.095727153Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-dc4ng to exist\\nMachine j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qxkjhfl8b89k--managed/providers/Microsoft.Compute/virtualMachines/j1e2a9k7q9d5m2t-np-1-xwkgq-j9d76 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0669824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fb248cdb-708c-4e1a-a191-a9d66edb0204\n correlation ID: 22bcc657-19c6-431f-9432-afe9f54b1a18\n client request ID: 0f3d0f1f-4c8a-4108-a1df-7f5cda2678a1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110960916789989376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7463,"pr_state":"open","pr_sha":"47842e47b8d78b53999bc8c4ef95f6c97a432ace","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T05:07:56Z","started_at":"2026-10-16T05:07:56Z","completed_at":"2026-10-16T07:27:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T07:39:05.389806398Z","timing_metadata_checked_at":"2026-10-16T07:39:05.389806398Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T07:39:05.394156284Z","region_metadata_checked_at":"2026-10-16T07:39:05.394156284Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110949609818820608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7561,"pr_state":"open","pr_sha":"64bbea1d9c2a33444e172687d4895cd34ea82928","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T04:23:01Z","started_at":"2026-10-16T04:23:01Z","completed_at":"2026-10-16T06:36:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T06:47:38.652413192Z","timing_metadata_checked_at":"2026-10-16T06:47:38.652413192Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T06:47:38.656675176Z","region_metadata_checked_at":"2026-10-16T06:47:38.656675176Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f8c4c2db7a050c8f6eda6bee582ce592948d73cbbff7c1a0bbfc333fd8f75df6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110949609818820608","occurred_at":"2026-10-16T04:23:01Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T05:28:35Z\n Ended: 2026-10-16T05:43:39Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8820608-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110946452283854848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"closed","pr_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","final_merged_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T04:10:28Z","started_at":"2026-10-16T04:10:28Z","completed_at":"2026-10-16T04:53:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T05:10:56.250145012Z","timing_metadata_checked_at":"2026-10-16T05:10:56.250145012Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T05:10:56.253470101Z","region_metadata_checked_at":"2026-10-16T05:10:56.253470101Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"08b4374e6b238d9b473ec523b5d908432ea963551fe8281a28505dd90dfd8215","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110946452283854848","occurred_at":"2026-10-16T04:10:28Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T04:32:52.168Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T04:32:53.669Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-16T04:32:54.816Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3854848%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a\ntime=2026-10-16T04:48:28.496Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3854848/providers/Microsoft.Resources/deployments/52b936b326777547bb7821b194e68c26b111cb76ef998bdad29331fe388cab6a/operationStatuses/08584094817114313637\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3854848/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3854848/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3854848\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110943321047175168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7132,"pr_state":"closed","pr_sha":"823aadd55f050b7e4bb63d67795c10122e0d976f","final_merged_sha":"9f60977a89121124186b3a9255a54b3e5c0b283b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T03:57:57Z","started_at":"2026-10-16T03:57:57Z","completed_at":"2026-10-16T06:21:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T06:37:37.458131929Z","timing_metadata_checked_at":"2026-10-16T06:37:37.458131929Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T06:37:37.461941316Z","region_metadata_checked_at":"2026-10-16T06:37:37.461941316Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"11b9101bf62c1515514764bb3cfdc26af78cb7778ad4e6b4a4ef051efe7bf8a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110943321047175168","occurred_at":"2026-10-16T03:57:57Z","failed_at":"alert","signature_id":"478d3a30f078ef1e264b281b0279bc5e3d4879af8a4361b3ba264c28c23360e4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 10 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-fksqp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-fksqp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-x9rnt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-x9rnt has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-v5kgz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-v5kgz has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-tmxw7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-tmxw7 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-2s6gb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-2s6gb has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-5gsvd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-5gsvd has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-wh5lq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-wh5lq has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-5ljrz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-5ljrz has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-jncwh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-jncwh has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-16T05:20:25Z\n Ended: 2026-10-16T05:44:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7175168-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z\", pod=\"ignition-server-557b99d5f7-x2w2x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfm2t6lac4bm28asp1id21nho63gnk4-t4m3u7u5b1l6z7z/ignition-server-557b99d5f7-x2w2x has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"3960ae9de201c2faeb7f239a673f4e13467b21ce42e45f799b60e5b6c49a0c23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110943321047175168","occurred_at":"2026-10-16T03:57:57Z","failed_at":"e2e","signature_id":"3f087f03f689e64c84aa13709a63b29a529c080753b6b960c4759f2524e920f5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x34648f7ca8a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x34648f7ca880\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x34648e72c7d0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x34648f7ca860\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x3464907e0690\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x34648f7ca8a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.004s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x34648f7ca880\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x34648e72c7d0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x34648f7ca860\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ggkzx97phmqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:59:51.6607748Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x3464907e0690\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110939401772077056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"closed","pr_sha":"d75d1168a97469ec8843d6c742ff668fb31c1216","final_merged_sha":"8849e5d7c268e9779b73b664bf5de466df0d0ef6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T03:42:27Z","started_at":"2026-10-16T03:42:27Z","completed_at":"2026-10-16T05:47:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T05:55:48.915150004Z","timing_metadata_checked_at":"2026-10-16T05:55:48.915150004Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T05:55:48.921061882Z","region_metadata_checked_at":"2026-10-16T05:55:48.921061882Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"69dab2b82091e5036b0f7e3810dfdc152d77c9df4e8ecf39b45eb5942505705c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110939401772077056","occurred_at":"2026-10-16T03:42:27Z","failed_at":"e2e","signature_id":"8f88f36aa920c3c11d09edc503e4d36cde94fa4e3555afb3adfaf72dcfd404e5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x16b0282ea00\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x16b0282e9e0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x16b028e0910\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x16b0282e9c0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x16b04458db0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x16b0282ea00\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x16b0282e9e0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x16b028e0910\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x16b0282e9c0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-99b8dvblsqkq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-16T04:23:19.3198868Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x16b04458db0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110923393426526208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"closed","pr_sha":"d75d1168a97469ec8843d6c742ff668fb31c1216","final_merged_sha":"8849e5d7c268e9779b73b664bf5de466df0d0ef6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T02:38:50Z","started_at":"2026-10-16T02:38:50Z","completed_at":"2026-10-16T03:39:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:47:25.953700605Z","timing_metadata_checked_at":"2026-10-16T03:47:25.953700605Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:47:25.956489596Z","region_metadata_checked_at":"2026-10-16T03:47:25.956489596Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"44508eb2528a429a753efa1c9b26f3a0e7213cb2220c32f875c57d52c4023371","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110923393426526208","occurred_at":"2026-10-16T02:38:50Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T03:17:16.536Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T03:17:18.501Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-16T03:17:19.523Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6526208%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57\ntime=2026-10-16T03:35:23.849Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6526208/providers/Microsoft.Resources/deployments/23386a13ae4b9c10e1167d28d0eb0e67c647056f11a0c2bc948bc314d383ed57/operationStatuses/08584094862465785031\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6526208/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6526208/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j6526208\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110912562055876608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7561,"pr_state":"open","pr_sha":"8a87d030ad8ca154cff896e6a20ddfc6a93ed18b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T01:55:48Z","started_at":"2026-10-16T01:55:48Z","completed_at":"2026-10-16T02:56:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:15:29.11484096Z","timing_metadata_checked_at":"2026-10-16T03:15:29.11484096Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:15:29.11780835Z","region_metadata_checked_at":"2026-10-16T03:15:29.11780835Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3c70de189e11fe1c799c8d81b00f4f613e1a8f3c098a80b8cfe10d221612b30b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7561/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110912562055876608","occurred_at":"2026-10-16T01:55:48Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T02:17:33.774Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T02:17:35.249Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-16T02:17:36.184Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5876608%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8\ntime=2026-10-16T02:33:38.292Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5876608/providers/Microsoft.Resources/deployments/11568e45e85c71193ac85f28e42fe8900978b11d9ee87376f97b514ef517dce8/operationStatuses/08584094898298983239\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5876608/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5876608/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5876608\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110910911651778560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"closed","pr_sha":"d75d1168a97469ec8843d6c742ff668fb31c1216","final_merged_sha":"8849e5d7c268e9779b73b664bf5de466df0d0ef6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T01:49:08Z","started_at":"2026-10-16T01:49:08Z","completed_at":"2026-10-16T02:33:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T02:54:02.218103853Z","timing_metadata_checked_at":"2026-10-16T02:54:02.218103853Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T02:54:02.221535642Z","region_metadata_checked_at":"2026-10-16T02:54:02.221535642Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"86becee4b537361367e54f61a49b4026aaa85a140d41fc932d4cd9ca25bb9aeb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110910911651778560","occurred_at":"2026-10-16T01:49:08Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T02:12:04.133Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T02:12:05.706Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-16T02:12:06.465Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fd4c8415653ce521dcac897b418e014c2ef331c8977fc03faf9a6142657d4dcd portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1778560%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffd4c8415653ce521dcac897b418e014c2ef331c8977fc03faf9a6142657d4dcd\ntime=2026-10-16T02:29:09.429Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1778560/providers/Microsoft.Resources/deployments/fd4c8415653ce521dcac897b418e014c2ef331c8977fc03faf9a6142657d4dcd/operationStatuses/08584094901594831073\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1778560/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1778560/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1778560\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110909868377378816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"closed","pr_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","final_merged_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-16T01:45:06Z","started_at":"2026-10-16T01:45:06Z","completed_at":"2026-10-16T02:55:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:15:29.085813956Z","timing_metadata_checked_at":"2026-10-16T03:15:29.085813956Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:15:29.089638943Z","region_metadata_checked_at":"2026-10-16T03:15:29.089638943Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0f95a038b20e62c5c119e331dc4599460e81c3aee1173d1d5e8270de4563487e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110909868377378816","occurred_at":"2026-10-16T01:45:06Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-16T02:15:07.162Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-16T02:15:08.937Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-16T02:15:09.609Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d5db28dfe0cc3780f80baae497ef5a933a88e4b7ad9d7697cd455d31b3b34457 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7378816%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd5db28dfe0cc3780f80baae497ef5a933a88e4b7ad9d7697cd455d31b3b34457\ntime=2026-10-16T02:30:42.385Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7378816/providers/Microsoft.Resources/deployments/d5db28dfe0cc3780f80baae497ef5a933a88e4b7ad9d7697cd455d31b3b34457/operationStatuses/08584094899762652758\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7378816/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7378816/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7378816\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7560/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110901774524092416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7560,"pr_state":"closed","pr_sha":"2b31066ac8edf101692a31d9897cd86eab65d295","final_merged_sha":"2b31066ac8edf101692a31d9897cd86eab65d295","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T01:12:56Z","started_at":"2026-10-16T01:12:56Z","completed_at":"2026-10-16T03:30:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:37:13.132951519Z","timing_metadata_checked_at":"2026-10-16T03:37:13.132951519Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:37:13.13574291Z","region_metadata_checked_at":"2026-10-16T03:37:13.13574291Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7570/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110899455954784256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7570,"pr_state":"closed","pr_sha":"a1acc85a713d8ae3e35a6f82d1bb43dccd30c1ea","final_merged_sha":"a1acc85a713d8ae3e35a6f82d1bb43dccd30c1ea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-16T01:03:43Z","started_at":"2026-10-16T01:03:43Z","completed_at":"2026-10-16T03:26:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:37:13.349783054Z","timing_metadata_checked_at":"2026-10-16T03:37:13.349783054Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:37:13.352358447Z","region_metadata_checked_at":"2026-10-16T03:37:13.352358447Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110896106962423808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7472,"pr_state":"closed","pr_sha":"e575472d09c149fa2a0e7534e42f1091494ed5a1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T00:50:25Z","started_at":"2026-10-16T00:50:25Z","completed_at":"2026-10-16T03:29:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:37:12.931510336Z","timing_metadata_checked_at":"2026-10-16T03:37:12.931510336Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:37:12.934031929Z","region_metadata_checked_at":"2026-10-16T03:37:12.934031929Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"4d8886434a6e285cbc850502d4c2b2a2dd8369cd0a392a3fc0c72d9a02bc238e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110896106962423808","occurred_at":"2026-10-16T00:50:25Z","failed_at":"alert","signature_id":"847391a6f039d89e83174f4d45c5d987309f37b3c9a63e6f0f0cdd1d71ddfba6","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T02:15:00Z\n Ended: 2026-10-16T02:21:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g\", pod=\"ignition-server-proxy-57c48555fb-9c282\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/ignition-server-proxy-57c48555fb-9c282 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T02:15:00Z\n Ended: 2026-10-16T02:21:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g\", pod=\"kube-controller-manager-54f5967dd5-6m76d\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/kube-controller-manager-54f5967dd5-6m76d has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-16T02:15:00Z\n Ended: 2026-10-16T02:21:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g\", pod=\"ignition-server-proxy-57c48555fb-skdc9\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/ignition-server-proxy-57c48555fb-skdc9 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-16T02:15:00Z\n Ended: 2026-10-16T02:28:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q\", pod=\"azure-disk-csi-driver-operator-799c47857f-c7ffd\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q/azure-disk-csi-driver-operator-799c47857f-c7ffd has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-16T02:15:00Z\n Ended: 2026-10-16T02:21:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g\", pod=\"kube-controller-manager-54f5967dd5-f9767\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/kube-controller-manager-54f5967dd5-f9767 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-16T02:15:00Z\n Ended: 2026-10-16T02:21:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g\", pod=\"kube-controller-manager-54f5967dd5-58bpn\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/kube-controller-manager-54f5967dd5-58bpn has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-16T02:15:01Z\n Ended: 2026-10-16T02:28:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q\", pod=\"csi-snapshot-controller-645c7d8759-992mc\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q/csi-snapshot-controller-645c7d8759-992mc has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-16T02:15:01Z\n Ended: 2026-10-16T02:21:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g\", pod=\"ignition-server-proxy-57c48555fb-vr7j9\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/ignition-server-proxy-57c48555fb-vr7j9 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-16T02:15:01Z\n Ended: 2026-10-16T02:21:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g\", pod=\"ignition-server-proxy-57c48555fb-2flbm\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8nks8u9ubtqctv7kgrcktgoq2usd-d1b5d8g4h2n5k7g/ignition-server-proxy-57c48555fb-2flbm has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-16T02:15:01Z\n Ended: 2026-10-16T02:28:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q\", pod=\"csi-snapshot-controller-645c7d8759-lwl76\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q/csi-snapshot-controller-645c7d8759-lwl76 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-16T02:15:01Z\n Ended: 2026-10-16T02:28:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423808-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q\", pod=\"azure-file-csi-driver-operator-75d8977b68-cmb5g\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj8l4ndnv3djlrj2jshom2gcgub3ri-t5r1c4a6r1f3a8q/azure-file-csi-driver-operator-75d8977b68-cmb5g has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"fea53aee71a5c7f7dd07790ff0f9dcb2e4ccf3a458258c8ba4d059fcb3b0b165","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7472/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110896106962423808","occurred_at":"2026-10-16T00:50:25Z","failed_at":"alert","signature_id":"b17a76908c996524ea705a1504c61f6666570717cb26311f2fea20b9e82bf822","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T02:38:55Z\n Ended: 2026-10-16T03:16:55Z\n Severity: Sev3\n Labels: alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j2423808-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj30j2k6sdjh9r7vsbj6npes2m40m2-p2r6x1t9q6d6f9l\", region=\"centralus\", severity=\"warning\"\n Description: 1 cluster operator(s) on hosted cluster ocm-arohcpci01-2tfj30j2k6sdjh9r7vsbj6npes2m40m2-p2r6x1t9q6d6f9l (management cluster ci01-j2423808-mgmt-1) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110895015453528064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7471,"pr_state":"closed","pr_sha":"5a59ae997b7e22e230920daa62a60fef4b4ecc9e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T00:46:04Z","started_at":"2026-10-16T00:46:04Z","completed_at":"2026-10-16T03:27:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:37:12.614335408Z","timing_metadata_checked_at":"2026-10-16T03:37:12.614335408Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:37:12.617763098Z","region_metadata_checked_at":"2026-10-16T03:37:12.617763098Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"a78f3c4f7395b1f18c6bc56b9f1840068fc5019eba8bbf0160158f24332f040c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110895015453528064","occurred_at":"2026-10-16T00:46:04Z","failed_at":"alert","signature_id":"5d614e354a186d8a4f21f749eb16466b0722b8015b284c219df8c2b54634da4e","test_name":"[aro-hcp-observability] [hcp] alert HCPClusterOperatorUnavailable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T02:40:29Z\n Ended: 2026-10-16T03:15:31Z\n Severity: Sev3\n Labels: alertname=\"HCPClusterOperatorUnavailable\", cluster=\"ci01-j3528064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3qrd1lfn90p1i58kr795ndsi2u3g-f4z5z8b0a1u0o2v\", region=\"westus3\", severity=\"warning\"\n Description: 2 cluster operator(s) on hosted cluster ocm-arohcpci01-2tfj3qrd1lfn90p1i58kr795ndsi2u3g-f4z5z8b0a1u0o2v (management cluster ci01-j3528064-mgmt-2) have been reporting Available=false for more than 30 minutes. The version and console operators are excluded from this alert; the affected cluster has worker nodes present. An unavailable operator means the component it manages is down, not merely degraded.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2","failure_pattern":"alert [hcp] HCPClusterOperatorUnavailable fired","search_query":"alert [hcp] HCPClusterOperatorUnavailable fired"}},{"row_id":"de5d8726d8613540f86044188a023ad26248de84f5094376259c1613044d12b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7471/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110895015453528064","occurred_at":"2026-10-16T00:46:04Z","failed_at":"alert","signature_id":"c6522e5d94df9f7aede7ecf7a33352ae735ba2dd77d94a580cfe6f7e5bb7305c","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 82 time(s) (64 unknown, 18 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T02:10:05Z\n Ended: 2026-10-16T02:34:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-m9lxt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-m9lxt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T02:10:05Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-468dp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-468dp has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-16T02:10:05Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-mcp7n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-mcp7n has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-2lx2c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-2lx2c has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-nw6s6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-nw6s6 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-7bc8c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-7bc8c has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"network-node-identity-5b8749cb94-wvtfh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-wvtfh has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-55wn4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-55wn4 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-n2fw5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-n2fw5 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-45l5z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-45l5z has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-5k68x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5k68x has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-59vdg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-59vdg has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-d54pm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-d54pm has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-xqpw9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-xqpw9 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-k2h6l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-k2h6l has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-mntnm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-mntnm has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-5bbbw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5bbbw has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-bjfl2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-bjfl2 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"network-node-identity-5b8749cb94-xmxx2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-xmxx2 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-dfxm6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-dfxm6 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-nwmdr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-nwmdr has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-ddvnx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-ddvnx has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-mnjm6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-mnjm6 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-tkqrb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-tkqrb has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-blbx4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-blbx4 has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-gzwn6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-gzwn6 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-l9ch7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-l9ch7 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-nq9pw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-nq9pw has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-xtf8z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-xtf8z has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-gm47p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-gm47p has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-bclsd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-bclsd has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-ht9sj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-ht9sj has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-p4gcc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-p4gcc has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-qr5vd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-qr5vd has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-jncsq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-jncsq has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-46plt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-46plt has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-l2zcs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-l2zcs has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-l4r92\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-l4r92 has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"network-node-identity-5b8749cb94-v996p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-v996p has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"network-node-identity-5b8749cb94-czbkf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-czbkf has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-dj84f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-dj84f has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-p8zp2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-p8zp2 has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-lmxtt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-lmxtt has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-q7sq8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-q7sq8 has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-bt4kv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-bt4kv has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-ztqwt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-ztqwt has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"network-node-identity-5b8749cb94-vtzrd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-vtzrd has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-hsncg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-hsncg has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-bzqvx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-bzqvx has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-sg28h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-sg28h has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"network-node-identity-5b8749cb94-bq57x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-bq57x has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-zr8bf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-zr8bf has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-sbsz2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-sbsz2 has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-5w62k\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5w62k has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-ssb27\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-ssb27 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-2lnh7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-2lnh7 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-9wkkg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-9wkkg has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-flxcj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-flxcj has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-5dvp5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5dvp5 has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-6gc2q\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-6gc2q has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-s9ccj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-s9ccj has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-5t86z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-5t86z has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"ovnkube-control-plane-6685c9694f-s6t9p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/ovnkube-control-plane-6685c9694f-s6t9p has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-10-16T02:10:06Z\n Ended: 2026-10-16T02:34:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3528064-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e\", pod=\"network-node-identity-5b8749cb94-2v49b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj9vdu08uh5snrrsve7h5f9qid50fq-f4e1p2h3v8l8m6e/network-node-identity-5b8749cb94-2v49b has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","c65eaecf2bcf9cab816e9e9171a4f334676466d8dfb2f88bb604984fc8e6cfe2"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110894017456640000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7470,"pr_state":"closed","pr_sha":"3238baa0870b6e0d36385a8015571c33a7f7514d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T00:42:06Z","started_at":"2026-10-16T00:42:06Z","completed_at":"2026-10-16T03:19:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:37:12.304971257Z","timing_metadata_checked_at":"2026-10-16T03:37:12.304971257Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:37:12.307980447Z","region_metadata_checked_at":"2026-10-16T03:37:12.307980447Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"662e00a6bdbf3c60a15cd6b551b53309969bedbf5b358e5b6b4010a9702f0ba4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7470/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110894017456640000","occurred_at":"2026-10-16T00:42:06Z","failed_at":"alert","signature_id":"48ac6e834ed843ec8689357e4df1c77bc51934e184727a31bf74980e2baec415","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s) (3 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T02:01:55Z\n Ended: 2026-10-16T02:17:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6640000-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster\", pod=\"ignition-server-proxy-56c8c65f87-54tt4\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster/ignition-server-proxy-56c8c65f87-54tt4 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T02:01:55Z\n Ended: 2026-10-16T02:17:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6640000-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster\", pod=\"ignition-server-proxy-56c8c65f87-nhp86\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster/ignition-server-proxy-56c8c65f87-nhp86 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-16T02:01:55Z\n Ended: 2026-10-16T02:17:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6640000-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster\", pod=\"ignition-server-proxy-56c8c65f87-27bzh\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj6e0g80idsan97hfiqksfb6u2a1jr-cilium-cluster/ignition-server-proxy-56c8c65f87-27bzh has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110892726588280832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7467,"pr_state":"closed","pr_sha":"d1e52157dd546818357d333f89a1d415b8664db5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-16T00:36:59Z","started_at":"2026-10-16T00:36:59Z","completed_at":"2026-10-16T03:05:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:26:16.124712639Z","timing_metadata_checked_at":"2026-10-16T03:26:16.124712639Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:26:16.128720626Z","region_metadata_checked_at":"2026-10-16T03:26:16.128720626Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"885bc5b20aceca2ff6c198eef4290c0bac149e055799e066e23c5223dbf0edaa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7467/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110892726588280832","occurred_at":"2026-10-16T00:36:59Z","failed_at":"alert","signature_id":"f98f21b954d35d911f699ca50bd748cba1e83339b2039f6a9126e37d1ffd7622","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 54 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-8prks\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-8prks has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-nb952\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-nb952 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-wb8jp\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-wb8jp has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-b8j7b\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-b8j7b has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-tdtms\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-tdtms has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-b6c6j\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-b6c6j has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-2pn5b\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-2pn5b has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-t5wcc\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-t5wcc has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-vdvhv\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-vdvhv has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-rlpxn\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-rlpxn has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-bvbtx\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-bvbtx has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k\", pod=\"network-node-identity-6cc47c64d4-7pj68\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k/network-node-identity-6cc47c64d4-7pj68 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k\", pod=\"network-node-identity-6cc47c64d4-rxvck\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k/network-node-identity-6cc47c64d4-rxvck has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k\", pod=\"network-node-identity-6cc47c64d4-x7gdj\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k/network-node-identity-6cc47c64d4-x7gdj has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-cbqv5\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-cbqv5 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-l8frm\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-l8frm has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-cnrt7\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-cnrt7 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-fzpnb\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-fzpnb has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k\", pod=\"network-node-identity-6cc47c64d4-z5fsz\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3co4h1r2t5ii0gmn6ldk0re4j08c-u1k7f1g2b4f1i7k/network-node-identity-6cc47c64d4-z5fsz has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-qgw4k\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-qgw4k has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:12:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-6zh7n\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-6zh7n has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-ngdpw\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-ngdpw has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-llxb8\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-llxb8 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-16T01:55:18Z\n Ended: 2026-10-16T02:09:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n\", pod=\"network-node-identity-6c9c9b694-c7lsz\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3c8sksamhs9a12k0djniaa5mvs44-h9m4e7x1i7n0l4n/network-node-identity-6c9c9b694-c7lsz has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-49gpg\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-49gpg has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-jbs6b\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-jbs6b has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-nvqxc\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-nvqxc has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-7s5p2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-7s5p2 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-f5k6n\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-f5k6n has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-pg6x2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-pg6x2 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-jgx64\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-jgx64 has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-fm5rr\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-fm5rr has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-xb9s2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-xb9s2 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-c7r7c\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-c7r7c has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-qch8q\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-qch8q has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-9bsh7\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-9bsh7 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-xmx8g\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-xmx8g has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-9c99d\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-9c99d has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-tht7j\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-tht7j has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-9rc8c\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-9rc8c has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-bsw8c\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-bsw8c has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-7dr6c\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-7dr6c has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-8kmpk\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-8kmpk has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-r8vf2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-r8vf2 has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-2pf6l\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-2pf6l has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-9zbkd\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-9zbkd has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-zjwxx\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-zjwxx has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-xdz7r\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-xdz7r has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-wg7qs\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-wg7qs has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-8zmzm\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-8zmzm has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-68vhm\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-68vhm has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-8p9wj\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-8p9wj has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-8dqjv\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-8dqjv has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-16T01:56:18Z\n Ended: 2026-10-16T02:19:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8280832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e\", pod=\"network-node-identity-56cf76dccd-84lvn\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tfj3csrl0e17eh7pek2o6u5251hsqut-x1g3j2x1f6w2y3e/network-node-identity-56cf76dccd-84lvn has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7468/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110891816172654592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7468,"pr_state":"closed","pr_sha":"20c9af71829f0b213a3dbf25d2cb69b7cc96b6d3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-16T00:33:22Z","started_at":"2026-10-16T00:33:22Z","completed_at":"2026-10-16T03:26:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-16T03:37:11.972761275Z","timing_metadata_checked_at":"2026-10-16T03:37:11.972761275Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-16T03:37:11.975397367Z","region_metadata_checked_at":"2026-10-16T03:37:11.975397367Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}