{"meta":{"date":"2026-10-19","timezone":"UTC","generated_at":"2026-10-31T05:51:38Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":92,"failed_runs":75,"runs_with_occurrences":75,"runs_with_failure_pattern_matches":68,"runs_with_unmatched_occurrences":7,"runs_without_occurrence_rows":17,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112324326735745024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7278,"pr_state":"open","pr_sha":"ccc7e6ef7d4580c43fce4687e41935bc8a151608","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T23:25:39Z","started_at":"2026-10-19T23:25:39Z","completed_at":"2026-10-19T23:46:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:57:15.839968164Z","timing_metadata_checked_at":"2026-10-19T23:57:15.839968164Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-19T23:57:15.843367953Z","region_metadata_checked_at":"2026-10-20T00:58:57.01472139Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9f82fec351e347708c7d9b8125e43315392a6727e2f1c41efee8509e1aeccbce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112324326735745024","occurred_at":"2026-10-19T23:25:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7623/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112324186226561024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7623,"pr_state":"closed","pr_sha":"f3a39537683fe0b598985b1face8ba1a19de80b5","final_merged_sha":"f3a39537683fe0b598985b1face8ba1a19de80b5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T23:25:05Z","started_at":"2026-10-19T23:25:05Z","completed_at":"2026-10-19T23:48:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:57:16.642574077Z","timing_metadata_checked_at":"2026-10-19T23:57:16.642574077Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-19T23:57:16.645800266Z","region_metadata_checked_at":"2026-10-20T00:58:57.20458622Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1fe5b418affa61d50db0892dd9c27bd04c8016b095540b18a108482000d7c9d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7623/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112324186226561024","occurred_at":"2026-10-19T23:25:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112322496756715520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"91c333f0a7bc8265681ee04a0f45834a0a141b74","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T23:18:22Z","started_at":"2026-10-19T23:18:22Z","completed_at":"2026-10-20T00:21:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T00:28:28.68540499Z","timing_metadata_checked_at":"2026-10-20T00:28:28.68540499Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T00:28:28.690149875Z","region_metadata_checked_at":"2026-10-20T00:28:28.690149875Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a287c1042f5adf1897c353a911bc870a5078f0ce8cbc4dae3bad352ae246b3e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112322496756715520","occurred_at":"2026-10-19T23:18:22Z","failed_at":"provision","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-20T00:14:03.097Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-20T00:14:03.476Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-20T00:14:03.476Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-20T00:14:03.887Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-20T00:14:04.156Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-20T00:14:04.156Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-20T00:14:04.247Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-20T00:15:01.574Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-20T00:15:01.574Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112322180745269248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"open","pr_sha":"fe5aebe5fab7b62a83cda503dc4a453fc83e909b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T23:17:07Z","started_at":"2026-10-19T23:17:07Z","completed_at":"2026-10-19T23:45:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:57:16.334935407Z","timing_metadata_checked_at":"2026-10-19T23:57:16.334935407Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-19T23:57:16.338950594Z","region_metadata_checked_at":"2026-10-20T00:58:57.105917516Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2e8ba5d3521eaa56c34725b3b1c1d26c786985b8d857f7f3d9fe900513b39135","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112322180745269248","occurred_at":"2026-10-19T23:17:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321281692012544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7621,"pr_state":"closed","pr_sha":"774e1807a32ee4f1cc43d8ef284e32ac705edf07","final_merged_sha":"774e1807a32ee4f1cc43d8ef284e32ac705edf07","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T23:13:32Z","started_at":"2026-10-19T23:13:32Z","completed_at":"2026-10-20T01:50:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T02:00:37.117580372Z","timing_metadata_checked_at":"2026-10-20T02:00:37.117580372Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T02:00:37.127966937Z","region_metadata_checked_at":"2026-10-20T02:00:37.127966937Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1f7d3c369ae9b1db547c198df0d3d59f163761fec95dc8e755207ff63fe8ab58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321281692012544","occurred_at":"2026-10-19T23:13:32Z","failed_at":"e2e","signature_id":"e1dc04bb7befa9011c60269ec95a0b379daa505a076d651f252178142e2c1a97","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc80b6d518e0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.005s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc80b6d518e0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"3121fd5c2bb01e6c6595a29b3d8bd6d24676c7f8aed388cf6665bb504239bb81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321281692012544","occurred_at":"2026-10-19T23:13:32Z","failed_at":"e2e","signature_id":"1c4f6dbd6e184b39b789fddbe55d13e4716fae9f427dad99450ca3f073344d50","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xbabb28f39e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-zk8hdsmg8dfw\" to finish creating: GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\",\n \"name\": \"7dd28f77-c5a7-447f-9ff3-55ca2c7472be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T00:39:28.091453996Z\",\n \"endTime\": \"2026-10-20T00:58:30.991397546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\n correlation ID: 8265cd70-ed50-4525-a687-45c965024ae6\n client request ID: b95e9e3d-a0f7-4ee8-8ac7-2ff5d62a1d3a\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xbabb28f39e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-zk8hdsmg8dfw\" to finish creating: GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\",\n \"name\": \"7dd28f77-c5a7-447f-9ff3-55ca2c7472be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T00:39:28.091453996Z\",\n \"endTime\": \"2026-10-20T00:58:30.991397546Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7dd28f77-c5a7-447f-9ff3-55ca2c7472be\n correlation ID: 8265cd70-ed50-4525-a687-45c965024ae6\n client request ID: b95e9e3d-a0f7-4ee8-8ac7-2ff5d62a1d3a","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp"}},{"row_id":"855cee0d31784f9c21e0cdf2e9aaa47d9fe639914785122c41ed08ef58fe666c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321281692012544","occurred_at":"2026-10-19T23:13:32Z","failed_at":"e2e","signature_id":"97312b6eb167e1169037b6c71d59e152ec18721dd3caa446c5b8a711966bf072","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1fe322c7e500\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2qqsdjg4hsrq\" to finish creating: GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\",\n \"name\": \"7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T00:36:56.91702797Z\",\n \"endTime\": \"2026-10-20T00:55:57.103083611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-6lglh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Compute/virtualMachines/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh to exist\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-gl748: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Compute/virtualMachines/d8b7p0j2z4w5h6j-np-1-j5lml-gl748 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-6lglh: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-2qqsdjg4hsrq--managed/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Network/networkInterfaces/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to htt... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\n correlation ID: 2083ceae-e353-4ba6-a1e3-ff92ee472926\n client request ID: 9c9e7813-8848-4608-b832-b1dbe65ca4ad\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1fe322c7e500\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2qqsdjg4hsrq\" to finish creating: GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\",\n \"name\": \"7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-20T00:36:56.91702797Z\",\n \"endTime\": \"2026-10-20T00:55:57.103083611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-6lglh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Compute/virtualMachines/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh to exist\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-gl748: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Compute/virtualMachines/d8b7p0j2z4w5h6j-np-1-j5lml-gl748 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d8b7p0j2z4w5h6j-np-1-j5lml-6lglh: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-2qqsdjg4hsrq--managed/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-2qqsdjg4hsrq--managed/providers/Microsoft.Network/networkInterfaces/d8b7p0j2z4w5h6j-np-1-j5lml-6lglh-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to htt... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2012544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b17b2a2-ba01-48a2-bbf6-109c6c980ff7\n correlation ID: 2083ceae-e353-4ba6-a1e3-ff92ee472926\n client request ID: 9c9e7813-8848-4608-b832-b1dbe65ca4ad\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321203636015104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7463,"pr_state":"closed","pr_sha":"47842e47b8d78b53999bc8c4ef95f6c97a432ace","final_merged_sha":"962fb789f66c3a28fca02c0fc2977023613c2549","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T23:12:44Z","started_at":"2026-10-19T23:12:44Z","completed_at":"2026-10-20T01:48:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T01:50:16.131037503Z","timing_metadata_checked_at":"2026-10-20T01:50:16.131037503Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T01:50:16.134452393Z","region_metadata_checked_at":"2026-10-20T01:50:16.134452393Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0477541dd3291e2687e530cd2aa3bffa3620d370f6ed619a83d26720cf1f087d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112321203636015104","occurred_at":"2026-10-19T23:12:44Z","failed_at":"e2e","signature_id":"59dc8637178db724ee538e803f30db4a90c31b43e902905b89afa3e5cf02ff0b","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xaeb08df4660\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.008s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xaeb08df4660\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"758edd41ad6b415076e0d38cb748e50ae9ddaac4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T22:39:56Z","started_at":"2026-10-19T22:39:56Z","completed_at":"2026-10-20T01:10:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T01:19:35.550455987Z","timing_metadata_checked_at":"2026-10-20T01:19:35.550455987Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T01:19:35.553784276Z","region_metadata_checked_at":"2026-10-20T01:19:35.553784276Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"313047d127a731e818beff53c17a983e01dd5941fd2821e1377d380cedeb2c83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","occurred_at":"2026-10-19T22:39:56Z","failed_at":"e2e","signature_id":"386c6869d89c07e4c74c5f01b6e960ca1675fb507e88a5f58cb3f2db3a845593","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1cde6af4c2e0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1cde6af4c2c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1cde6a0b6140\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1cde6af4c2a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1cde6c048750\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1cde6af4c2e0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | lookup kind=inner hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1cde6af4c2c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1cde6a0b6140\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1cde6af4c2a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-hx6p2lg8h8cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-19T23:40:50.5353725Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1cde6c048750\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}},{"row_id":"6cc396737fbbaac6f4d74e59d3d1883b3d94bf1aef5331c1201687a4f89badf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","occurred_at":"2026-10-19T22:39:56Z","failed_at":"alert","signature_id":"f0c211f0475f4d8abc5d3fd2c2049832c9ac8b9c7ee9259b6cfe55ec42e3266f","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-20T00:03:40Z\n Ended: 2026-10-20T00:18:42Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0435200-svc\", component=\"backend\", name=\"ingresscertificate\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue ingresscertificate has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"6e35daa68cdb59d2db55d0adc4b047c8fd33efbdc31dda368d1124a6fb70f7c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","occurred_at":"2026-10-19T22:39:56Z","failed_at":"e2e","signature_id":"6f7bc0014c51cfc8989a4b4711871dc6b12df8b82d9220ab61461f581c9560eb","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x17293bf48060\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-qw8hmnl947sz\" to finish creating: GET https://rp.j0435200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\",\n \"name\": \"2be64b2f-b78e-49f6-addd-04faf33378bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T23:54:37.149407446Z\",\n \"endTime\": \"2026-10-20T00:13:41.899452027Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0435200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\n correlation ID: a4d6efb0-533f-4043-ad67-651fe433c379\n client request ID: 6f8a1c34-923e-40e5-a5b3-733c67d8d88f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x17293bf48060\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-qw8hmnl947sz\" to finish creating: GET https://rp.j0435200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\",\n \"name\": \"2be64b2f-b78e-49f6-addd-04faf33378bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T23:54:37.149407446Z\",\n \"endTime\": \"2026-10-20T00:13:41.899452027Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0435200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2be64b2f-b78e-49f6-addd-04faf33378bd\n correlation ID: a4d6efb0-533f-4043-ad67-651fe433c379\n client request ID: 6f8a1c34-923e-40e5-a5b3-733c67d8d88f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7162f2eabce7641480f3270c39ccfb7bdba5abf11c6929a8f411eb67c9e6e641","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112312824880435200","occurred_at":"2026-10-19T22:39:56Z","failed_at":"e2e","signature_id":"af83b0a23ad7b87be6df260bf29be76315509cb2eae7c8baf1ce1bbbbc9b604b","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.009s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x207ea69dfda0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:491]: Timed out after 600.009s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0x207ea69dfda0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n {\n s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112311076581281792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7621,"pr_state":"closed","pr_sha":"774e1807a32ee4f1cc43d8ef284e32ac705edf07","final_merged_sha":"774e1807a32ee4f1cc43d8ef284e32ac705edf07","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T22:33:00Z","started_at":"2026-10-19T22:33:00Z","completed_at":"2026-10-19T23:12:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:26:02.025169258Z","timing_metadata_checked_at":"2026-10-19T23:26:02.025169258Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T23:26:02.028480348Z","region_metadata_checked_at":"2026-10-19T23:26:02.028480348Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"fb897374e5be2958ce688cdf50513202e76f6d2b7b3f4180c924b5df249881f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112311076581281792","occurred_at":"2026-10-19T22:33:00Z","failed_at":"provision","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T23:00:45.350Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-19T23:00:46.914Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-19T23:00:47.803Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f66b53e6663a909706c4cc56f884e08afa3f703badb52eda8425598d78f027f5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1281792%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff66b53e6663a909706c4cc56f884e08afa3f703badb52eda8425598d78f027f5\ntime=2026-10-19T23:08:19.085Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1281792/providers/Microsoft.Resources/deployments/f66b53e6663a909706c4cc56f884e08afa3f703badb52eda8425598d78f027f5/operationStatuses/08584091560383429326\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1281792/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1281792/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j1281792\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RequestTimeout\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Database account creation failed. Operation Id: 84ce1c4b-104f-4d59-ba97-ca7f687e5f6a, Error : Message: {\\\\\\\\\\\\\\\"Errors\\\\\\\\\\\\\\\":[\\\\\\\\\\\\\\\"The requested operation exceeded maximum alloted time. Learn more: https://aka.ms/cosmosdb-tsg-service-request-timeout\\\\\\\\\\\\\\\"]}\\\\\\\\r\\\\\\\\nActivityId: 50b97f78-d517-42ba-b6b3-2e3807c11c56, Request URI: /apps/56b56b41-1aa1-47de-9428-4a9b9ccbfcc7/services/5492004a-a82d-4653-81b2-013acecf0ba5/partitions/4d300830-d333-45bc-8ad6-8d4a8244255d/replicas/134308569966594579s, RequestStats: \\\\\\\\r\\\\\\\\nRequestStartTime: 2026-10-19T23:06:28.2874662Z, RequestEndTime: 2026-10-19T23:06:33.6942116Z, Number of regions attempted:1\\\\\\\\r\\\\\\\\n{\\\\\\\\\\\\\\\"systemHistory\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:05:35.8111686Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.582,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21738860.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0143,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:05:45.8134136Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.599,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21747896.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0589,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32764,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":27},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:05:55.8162151Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":1.543,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21739028.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0371,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32764,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:06:05.8318538Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.956,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21715984.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.3135,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:06:15.8338344Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.371,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21721028.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0646,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:06:25.8342074Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.996,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":21715496.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0674,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32}]}\\\\\\\\r\\\\\\\\nRequestStart: 2026-10-19T23:06:28.2874662Z; ResponseTime: 2026-10-19T23:06:33.6942116Z; StoreResult: StorePhysicalAddress: rntbd://cdb-ms-prod-centralus1-be312.documents.azure.com:11303/apps/56b56b41-1aa1-47de-9428-4a9b9ccbfcc7/services/5492004a-a82d-4653-81b2-013acecf0ba5/partitions/4d300830-d333-45bc-8ad6-8d4a8244255d/replicas/134308569966594579s, LSN: -1, GlobalCommittedLsn: -1, PartitionKeyRangeId: , IsValid: False, StatusCode: 408, SubStatusCode: 0, RequestCharge: 0, ItemLSN: -1, SessionToken: , UsingLocalLSN: False, TransportException: null, BELatencyMs: 5410.799, ActivityId: 50b97f78-d517-42ba-b6b3-2e3807c11c56, RetryAfterInMs: , globalNRegionCommittedGLSN: -1, ReplicaHealthStatuses: [(port: 11303 | status: Connected | lkt: 10/19/2026 11:06:33 PM)], TransportRequestTimeline: {\\\\\\\\\\\\\\\"requestTimeline\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Created\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:28.2874662Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0013},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ChannelAcquisitionStarted\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:28.2874675Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0046},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pipelined\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:28.2874721Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0625},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Transit Time\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:28.2875346Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 5411.5582},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Received\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:33.6990928Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0545},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Completed\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-19T23:06:33.6991473Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0}],\\\\\\\\\\\\\\\"serviceEndpointStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"inflightRequests\\\\\\\\\\\\\\\":1,\\\\\\\\\\\\\\\"openConnections\\\\\\\\\\\\\\\":1},\\\\\\\\\\\\\\\"connectionStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"waitforConnectionInit\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"callsPendingReceive\\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\\"lastSendAttempt\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:04:15.0012932Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastSend\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:04:15.0012932Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastReceive\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-19T23:04:16.0332114Z\\\\\\\\\\\\\\\"},\\\\\\\\\\\\\\\"requestSizeInBytes\\\\\\\\\\\\\\\":867,\\\\\\\\\\\\\\\"responseMetadataSizeInBytes\\\\\\\\\\\\\\\":30,\\\\\\\\\\\\\\\"responseBodySizeInBytes\\\\\\\\\\\\\\\":133};\\\\\\\\r\\\\\\\\n ResourceType: Replica, OperationType: ForceConfigRefresh\\\\\\\\r\\\\\\\\n, SDK: Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"835eba9e18d4ce34b2c45e7a932a8d1b8ec3d4cc9cdfed55b9778db69913e7ed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RequestTimeout; detail message Database account creation failed. Operation Id: \u003cuuid\u003e, Error : Message: {\"Errors\":[\"The requested operation exceeded maximum alloted time. Learn more: \u003curl\u003e; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["835eba9e18d4ce34b2c45e7a932a8d1b8ec3d4cc9cdfed55b9778db69913e7ed"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112306028279238656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7497,"pr_state":"open","pr_sha":"793a8fb63e193e71a604f2de4aca38eabbab8dc9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-19T22:12:56Z","started_at":"2026-10-19T22:12:56Z","completed_at":"2026-10-20T00:26:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T00:38:43.022712801Z","timing_metadata_checked_at":"2026-10-20T00:38:43.022712801Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T00:38:43.026733789Z","region_metadata_checked_at":"2026-10-20T00:38:43.026733789Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112306017495683072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7604,"pr_state":"closed","pr_sha":"35c1994a708e09591727ec4f56c0dbf1f09e0870","final_merged_sha":"35c1994a708e09591727ec4f56c0dbf1f09e0870","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T22:12:53Z","started_at":"2026-10-19T22:12:53Z","completed_at":"2026-10-20T00:25:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T00:38:43.186716188Z","timing_metadata_checked_at":"2026-10-20T00:38:43.186716188Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T00:38:43.190296977Z","region_metadata_checked_at":"2026-10-20T00:38:43.190296977Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112302759632965632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7613,"pr_state":"closed","pr_sha":"3dfb2822106e726cdf3917926cbfc66f8b8d39a9","final_merged_sha":"1ffb0dcbcc764e62c843aaed5a91e3d2ef0b37df","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T21:59:57Z","started_at":"2026-10-19T21:59:57Z","completed_at":"2026-10-20T00:15:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T00:28:28.91187059Z","timing_metadata_checked_at":"2026-10-20T00:28:28.91187059Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T00:28:28.914738681Z","region_metadata_checked_at":"2026-10-20T00:28:28.914738681Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0b80fa4b0448fa477de0027f497b47e5e388325f249d553faab81e689a3688c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112302759632965632","occurred_at":"2026-10-19T21:59:57Z","failed_at":"e2e","signature_id":"303f1c558a1c431ef13a787c38a150317636b1cd97abc6da971b7ee7e7fc947f","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x15ac4447a018\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x15ac43752020\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x15ac44e58300\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x15ac434e29a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x15ac4447a018\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x15ac43752020\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x15ac44e58300\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x15ac434e29a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rzzvn.apps.aro.cilium-cl.2ru7.j2965632.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112302045275885568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7480,"pr_state":"closed","pr_sha":"e3878843a9c9237f1f4ebb646d43a36b8b36eeca","final_merged_sha":"230a5d27c416299f628f0667ba08deeea65c4c88","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T21:57:06Z","started_at":"2026-10-19T21:57:06Z","completed_at":"2026-10-20T00:16:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-20T00:28:28.804504022Z","timing_metadata_checked_at":"2026-10-20T00:28:28.804504022Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-20T00:28:28.813005695Z","region_metadata_checked_at":"2026-10-20T00:28:28.813005695Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9197773a752098743ca4e3ff25cbb3b7d801f12e433e7c63a7292bc3f048a1d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112302045275885568","occurred_at":"2026-10-19T21:57:06Z","failed_at":"e2e","signature_id":"c1c99568d26a1ef70092e8ca69e04aa48a183d5691882b98ba81908b32e59ff0","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0051281e80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6h69n8f46csz\" to finish creating: GET https://rp.j5885568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\",\n \"name\": \"65c67213-a430-416b-a8e1-9c576702841f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T23:03:54.135232982Z\",\n \"endTime\": \"2026-10-19T23:22:56.513764235Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6h69n8f46csz--managed/providers/Microsoft.Compute/virtualMachines/q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6h69n8f46csz--managed/providers/Microsoft.Compute/virtualMachines/q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5885568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\n correlation ID: d01ca859-c416-4ccc-bd23-51544b0af025\n client request ID: 937df210-9f80-46da-9e07-0166c55c864d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d0051281e80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6h69n8f46csz\" to finish creating: GET https://rp.j5885568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\",\n \"name\": \"65c67213-a430-416b-a8e1-9c576702841f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T23:03:54.135232982Z\",\n \"endTime\": \"2026-10-19T23:22:56.513764235Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6h69n8f46csz--managed/providers/Microsoft.Compute/virtualMachines/q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-6h69n8f46csz--managed/providers/Microsoft.Compute/virtualMachines/q9d8h1b2n4i4y9z-np-1-wb6tr-5xthm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5885568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/65c67213-a430-416b-a8e1-9c576702841f\n correlation ID: d01ca859-c416-4ccc-bd23-51544b0af025\n client request ID: 937df210-9f80-46da-9e07-0166c55c864d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112293841544417280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7547,"pr_state":"open","pr_sha":"c9d64b4faa4707bbad6290be6e27f2921b99ea91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T21:24:30Z","started_at":"2026-10-19T21:24:30Z","completed_at":"2026-10-19T23:41:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:57:16.122445619Z","timing_metadata_checked_at":"2026-10-19T23:57:16.122445619Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T23:57:16.125864107Z","region_metadata_checked_at":"2026-10-19T23:57:16.125864107Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"304d8a9f9dd742b627bca809d88a866be491e3df488dd1d7340d4afb7f03ec64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112293841544417280","occurred_at":"2026-10-19T21:24:30Z","failed_at":"alert","signature_id":"4ece708f0a144a12dcdf372fed7f3d095b244cb8f59c1436a40f58700cd7033b","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T22:44:19Z\n Ended: 2026-10-19T22:50:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4417280-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti4iojordd6hm0nnh191a3fmg3e1aub-f4w6r1h1a4m1v8n\", pod=\"openshift-apiserver-7dc759866b-hr75b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti4iojordd6hm0nnh191a3fmg3e1aub-f4w6r1h1a4m1v8n/openshift-apiserver-7dc759866b-hr75b has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112290330521899008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7597,"pr_state":"closed","pr_sha":"d071a83f32db9a874ed681ae192ed18ce7bc5bf0","final_merged_sha":"322d4798e9f946e8950f4ab188bec64d82a6143a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T21:10:33Z","started_at":"2026-10-19T21:10:33Z","completed_at":"2026-10-19T23:39:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:46:37.065399968Z","timing_metadata_checked_at":"2026-10-19T23:46:37.065399968Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T23:46:37.069415057Z","region_metadata_checked_at":"2026-10-19T23:46:37.069415057Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0ff489e999c05a99850b62fdebd263621d04e75510ba207db67d10ca7f55f74c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112290330521899008","occurred_at":"2026-10-19T21:10:33Z","failed_at":"e2e","signature_id":"788f52be295e2db0098488b52e436a5a34110ee9698b2cbb0ae2a28aed2ec616","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-vkgl4p\" for cluster \"cluster-hs-pre-vkgl4p\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d7273ae000\u003e: \n failed to create NodePool np-vkgl4p: failed waiting for nodepool=\"np-vkgl4p\" for cluster \"cluster-hs-pre-vkgl4p\" in resourcegroup=\"rg-hs-pre--j799lrwqxz6j\" to finish creating: GET https://rp.j1899008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\",\n \"name\": \"c5e9078f-f162-40b9-b342-a3ebc6cb8256\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:21:51.863828648Z\",\n \"endTime\": \"2026-10-19T22:40:54.499105259Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/providers/Microsoft.Compute/virtualMachines/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98: Failed: virtualmachine failed to create or update. err: failed to get existing resource rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/providers/Microsoft.Compute/virtualMachines/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '10' seconds after additional tokens are ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1899008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\n correlation ID: 2e542bba-f131-4b5d-bf4c-a77a8bc95e2d\n client request ID: 83a4b938-e107-403c-83e6-0592ebdaeff2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-vkgl4p\" for cluster \"cluster-hs-pre-vkgl4p\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d7273ae000\u003e: \n failed to create NodePool np-vkgl4p: failed waiting for nodepool=\"np-vkgl4p\" for cluster \"cluster-hs-pre-vkgl4p\" in resourcegroup=\"rg-hs-pre--j799lrwqxz6j\" to finish creating: GET https://rp.j1899008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\",\n \"name\": \"c5e9078f-f162-40b9-b342-a3ebc6cb8256\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:21:51.863828648Z\",\n \"endTime\": \"2026-10-19T22:40:54.499105259Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/providers/Microsoft.Compute/virtualMachines/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98: Failed: virtualmachine failed to create or update. err: failed to get existing resource rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--j799lrwqxz6j-vkgl4p--managed/providers/Microsoft.Compute/virtualMachines/f2u7w5o4k6n1j9f-np-vkgl4p-qlncl-xgc98\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '10' seconds after additional tokens are ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1899008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c5e9078f-f162-40b9-b342-a3ebc6cb8256\n correlation ID: 2e542bba-f131-4b5d-bf4c-a77a8bc95e2d\n client request ID: 83a4b938-e107-403c-83e6-0592ebdaeff2\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ba247174d89d7a38b8deaaec8673bd9296102e3e139d937d6b4d7517e364180","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are ... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ba247174d89d7a38b8deaaec8673bd9296102e3e139d937d6b4d7517e364180"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112290291259019264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7609,"pr_state":"closed","pr_sha":"2f5ec471013bfa762d22c0066746cc060390b69c","final_merged_sha":"2f5ec471013bfa762d22c0066746cc060390b69c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T21:10:24Z","started_at":"2026-10-19T21:10:24Z","completed_at":"2026-10-19T23:38:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:46:37.195003801Z","timing_metadata_checked_at":"2026-10-19T23:46:37.195003801Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T23:46:37.198225791Z","region_metadata_checked_at":"2026-10-19T23:46:37.198225791Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7602,"pr_state":"closed","pr_sha":"480592cf094dfe5a02ccdacd0bc1ae203134ac73","final_merged_sha":"4c1deb264ba30a91ffe18fcc860eb45faa86f563","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T21:03:34Z","started_at":"2026-10-19T21:03:34Z","completed_at":"2026-10-19T23:24:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:36:16.283033503Z","timing_metadata_checked_at":"2026-10-19T23:36:16.283033503Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T23:36:16.286320893Z","region_metadata_checked_at":"2026-10-19T23:36:16.286320893Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"009588992ee71e9f0ee90412d0773f3537d2f2f8728cff8b255da7b5108f280b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","failed_at":"alert","signature_id":"5a6e4b1077d837f8d74d606b27a4e172480813383d0ec67371f063d887422ebe","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T22:40:56Z\n Ended: 2026-10-19T22:52:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frhvn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"15590e4c1e02150a5c6d9c169fc6f6c460199a7782e5fa2bc18952093f5dccbe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","failed_at":"e2e","signature_id":"d9779475fed33830e4dd8ea5b21ff77ea27177e1e5ee96139b3ecd5a818f11f6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-d9xldnlstk6j/cluster-candidate-5-0-tlxgzk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c66751baf40\u003e: \n failed to create HCP cluster cluster-candidate-5-0-tlxgzk: failed waiting for cluster=\"cluster-candidate-5-0-tlxgzk\" in resourcegroup=\"rg-candidate-5-0-d9xldnlstk6j\" to finish creating: GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\",\n \"name\": \"6dc91de3-05e9-401b-9139-c466ece83065\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:04:14.113898084Z\",\n \"endTime\": \"2026-10-19T22:33:23.409451279Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\n correlation ID: 2a2d3ba7-babe-4fe8-ae4e-5435562f5656\n client request ID: a7fdb086-12e2-40d2-b194-107f4afa279d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-d9xldnlstk6j/cluster-candidate-5-0-tlxgzk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2c66751baf40\u003e: \n failed to create HCP cluster cluster-candidate-5-0-tlxgzk: failed waiting for cluster=\"cluster-candidate-5-0-tlxgzk\" in resourcegroup=\"rg-candidate-5-0-d9xldnlstk6j\" to finish creating: GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\",\n \"name\": \"6dc91de3-05e9-401b-9139-c466ece83065\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:04:14.113898084Z\",\n \"endTime\": \"2026-10-19T22:33:23.409451279Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6dc91de3-05e9-401b-9139-c466ece83065\n correlation ID: 2a2d3ba7-babe-4fe8-ae4e-5435562f5656\n client request ID: a7fdb086-12e2-40d2-b194-107f4afa279d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4755f69a6e7c073e82a22c476e9890936ed2fbab2031bd12bb19dd8cf1abbe64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","failed_at":"alert","signature_id":"3eb395fbe55fbf76ecbc5b29bc93126f665b5c614d9cc51af8cf407abbc2351a","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T22:28:50Z\n Ended: 2026-10-19T22:55:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"43f3c967-dd75-47e1-aa51-7f18084ed726\"\n Description: Pod ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-19T22:29:50Z\n Ended: 2026-10-19T22:55:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"43f3c967-dd75-47e1-aa51-7f18084ed726\"\n Description: Pod ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"53ef3261957644fb89e20e456c4f95a62718c1c93f558d3b5a022dd13918003d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","failed_at":"alert","signature_id":"e78bd233dcf87e792b6e12ee3292aaea83b0ab781757136a8fe46caec4bd397e","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T22:25:56Z\n Ended: 2026-10-19T22:51:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frhvn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T22:25:56Z\n Ended: 2026-10-19T22:51:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8647552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8647552/providers/microsoft.monitor/accounts/hcps-j8647552\", namespace=\"ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-frhvn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ti4bopt6j217c6p864tnvp7ieejcmnb-k4p0h9n4o7o7c1n/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"d435e032cb9c33e0cc2e24f71aa02550dbb8605cced856765926a7283b767d8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","failed_at":"e2e","signature_id":"5006c2ff99f83ec65ef1f594a7caa27df5a4f407aeb1491b1d22e62a65fae644","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:180]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x222007964a08\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x222007b84d20\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x2220079649d8\u003e{\n errs: [\n \u003c*errors.errorString | 0x222008c33e60\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:180]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x222007964a08\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x222007b84d20\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x2220079649d8\u003e{\n errs: [\n \u003c*errors.errorString | 0x222008c33e60\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.19:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1"}},{"row_id":"e08a33697c22166ee8989bd27aced7032c49578b1c7c5c67027c7e3dcb1a2a87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112288570218647552","occurred_at":"2026-10-19T21:03:34Z","failed_at":"e2e","signature_id":"b7c7573f03c8b34839471a9a294fd246154a93ada5e8f67ea9a5a4f9e6427d6b","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-24dvjs\" for cluster \"cluster-hs-pre-24dvjs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d397c5fe960\u003e: \n failed to create NodePool np-24dvjs: failed waiting for nodepool=\"np-24dvjs\" for cluster \"cluster-hs-pre-24dvjs\" in resourcegroup=\"rg-hs-pre--qmdx974n6w26\" to finish creating: GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\",\n \"name\": \"8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:14:24.493403189Z\",\n \"endTime\": \"2026-10-19T22:33:34.182492247Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--qmdx974n6w26-24dvjs--managed/providers/Microsoft.Compute/virtualMachines/w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--qmdx974n6w26-24dvjs--managed/providers/Microsoft.Compute/virtualMachines/w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\n correlation ID: 33ce383b-ea0f-4d30-8902-6098f7bbfeb8\n client request ID: 9ce1f451-6c72-4c5f-8183-8c4490de781d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-24dvjs\" for cluster \"cluster-hs-pre-24dvjs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d397c5fe960\u003e: \n failed to create NodePool np-24dvjs: failed waiting for nodepool=\"np-24dvjs\" for cluster \"cluster-hs-pre-24dvjs\" in resourcegroup=\"rg-hs-pre--qmdx974n6w26\" to finish creating: GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\",\n \"name\": \"8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T22:14:24.493403189Z\",\n \"endTime\": \"2026-10-19T22:33:34.182492247Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--qmdx974n6w26-24dvjs--managed/providers/Microsoft.Compute/virtualMachines/w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--qmdx974n6w26-24dvjs--managed/providers/Microsoft.Compute/virtualMachines/w6l5b9y4j0d7a0b-np-24dvjs-sb457-th5dq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8647552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c4bd3c2-dba4-4fd4-adb1-10b5c21cbdf0\n correlation ID: 33ce383b-ea0f-4d30-8902-6098f7bbfeb8\n client request ID: 9ce1f451-6c72-4c5f-8183-8c4490de781d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112284210382245888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"1767bdfde51a636bd7588c375d91adc3d907bfe3","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T20:46:14Z","started_at":"2026-10-19T20:46:14Z","completed_at":"2026-10-19T21:05:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:20:42.228112092Z","timing_metadata_checked_at":"2026-10-19T21:20:42.228112092Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-19T21:20:42.234412271Z","region_metadata_checked_at":"2026-10-19T22:22:35.015604667Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"986636e26e175134e99f6209cbe8050e6a8e2af464d0aae72e84a7c1bdeb843a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112284210382245888","occurred_at":"2026-10-19T20:46:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112281126293737472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7612,"pr_state":"closed","pr_sha":"9cd3a4861b874682d79b475a28a57e4cc916d444","final_merged_sha":"9cd3a4861b874682d79b475a28a57e4cc916d444","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T20:33:59Z","started_at":"2026-10-19T20:33:59Z","completed_at":"2026-10-19T22:49:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:04:38.012269801Z","timing_metadata_checked_at":"2026-10-19T23:04:38.012269801Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T23:04:38.020615376Z","region_metadata_checked_at":"2026-10-19T23:04:38.020615376Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"262bff8cfbbef3f6416be8db765cbcb831c9ad2dd2f4fafdf5f8e9e3ca478637","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112281126293737472","occurred_at":"2026-10-19T20:33:59Z","failed_at":"alert","signature_id":"1584cff6139bca6b6622e28aa8cc37a138dc43ddd12fb3a8e76e579f7fdcf427","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 10 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T21:42:05Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-796b59578b-rbqnk\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-796b59578b-rbqnk has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T21:42:05Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-vr6v5\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-vr6v5 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T21:42:05Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-9vsw9\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-9vsw9 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3pc5270b8h60pst14u078uaplvhlq-q0m7n1d5p4o1u4j\", pod=\"azure-disk-csi-driver-controller-65f5bf78b-s9gq4\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3pc5270b8h60pst14u078uaplvhlq-q0m7n1d5p4o1u4j/azure-disk-csi-driver-controller-65f5bf78b-s9gq4 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-sb8lp\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-sb8lp has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-796b59578b-bjj96\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-796b59578b-bjj96 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-796b59578b-w4b2v\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-796b59578b-w4b2v has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-8qtrk\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-8qtrk has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-mkmmq\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-mkmmq has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T21:42:06Z\n Ended: 2026-10-19T22:08:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3737472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas\", pod=\"azure-disk-csi-driver-controller-68d8d85f58-rvxrh\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti3p9tdj7jhju8rr4baflukekfp093j-private-kas/azure-disk-csi-driver-controller-68d8d85f58-rvxrh has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112280966243291136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7065,"pr_state":"open","pr_sha":"b1c442f24e647a96a09fa2938c9466dbc0f43a55","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-19T20:33:21Z","started_at":"2026-10-19T20:33:21Z","completed_at":"2026-10-19T22:59:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T23:15:24.264757214Z","timing_metadata_checked_at":"2026-10-19T23:15:24.264757214Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T23:15:24.268573103Z","region_metadata_checked_at":"2026-10-19T23:15:24.268573103Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112275231841193984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7463,"pr_state":"closed","pr_sha":"47842e47b8d78b53999bc8c4ef95f6c97a432ace","final_merged_sha":"962fb789f66c3a28fca02c0fc2977023613c2549","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T20:10:34Z","started_at":"2026-10-19T20:10:34Z","completed_at":"2026-10-19T22:39:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T22:54:24.869575772Z","timing_metadata_checked_at":"2026-10-19T22:54:24.869575772Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T22:54:24.872986361Z","region_metadata_checked_at":"2026-10-19T22:54:24.872986361Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f808c86f7fc55b07173e521367fd0cd4b13d5b68aa2c6ebd686e8b0b46b6ce68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112275231841193984","occurred_at":"2026-10-19T20:10:34Z","failed_at":"e2e","signature_id":"a5f3943cfdc00ecde813bc0fb08ef3bd16b3ef30062a43d942be7abcdafd19ff","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xcdba5b46020\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp.j1193984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\",\n \"name\": \"f0e04584-5bfd-41bb-a2f1-00de7ac45771\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T21:20:53.14343965Z\",\n \"endTime\": \"2026-10-19T21:40:01.878429551Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y1d9g5x1u0o0j2g-np-1-klnmc-mrz7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y1d9g5x1u0o0j2g-np-1-klnmc-mrz7g: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1193984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\n correlation ID: 85a1b8fd-965d-4893-90c4-5bc86f87328d\n client request ID: 6c4ed168-8339-4ad9-9c7e-dcd04bbc9a76\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xcdba5b46020\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp.j1193984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\",\n \"name\": \"f0e04584-5bfd-41bb-a2f1-00de7ac45771\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T21:20:53.14343965Z\",\n \"endTime\": \"2026-10-19T21:40:01.878429551Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y1d9g5x1u0o0j2g-np-1-klnmc-mrz7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y1d9g5x1u0o0j2g-np-1-klnmc-mrz7g: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1193984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0e04584-5bfd-41bb-a2f1-00de7ac45771\n correlation ID: 85a1b8fd-965d-4893-90c4-5bc86f87328d\n client request ID: 6c4ed168-8339-4ad9-9c7e-dcd04bbc9a76","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112274862377537536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7604,"pr_state":"closed","pr_sha":"35c1994a708e09591727ec4f56c0dbf1f09e0870","final_merged_sha":"35c1994a708e09591727ec4f56c0dbf1f09e0870","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T20:09:05Z","started_at":"2026-10-19T20:09:05Z","completed_at":"2026-10-19T22:10:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T22:22:35.13363635Z","timing_metadata_checked_at":"2026-10-19T22:22:35.13363635Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T22:22:35.137221041Z","region_metadata_checked_at":"2026-10-19T22:22:35.137221041Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ad6bfe438cd607b23bba3bc25c1c0de56bbdb029ec96af66f0ac1191571e8ecb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112274862377537536","occurred_at":"2026-10-19T20:09:05Z","failed_at":"alert","signature_id":"dfee0196c4ef11f2d05adeebeabed98c6ade9a65dc79ffe19ad01f1c11c7117f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s) (1 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T20:56:48Z\n Ended: 2026-10-19T21:03:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7537536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti35rfutnepdsfn0vfq6aa39v1svufp-f2i3r5n2t0v2v3v\", pod=\"router-6dbb4f8499-ccrn9\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti35rfutnepdsfn0vfq6aa39v1svufp-f2i3r5n2t0v2v3v/router-6dbb4f8499-ccrn9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112270689380601856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"1767bdfde51a636bd7588c375d91adc3d907bfe3","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T19:52:24Z","started_at":"2026-10-19T19:52:24Z","completed_at":"2026-10-19T20:27:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T20:37:52.957502201Z","timing_metadata_checked_at":"2026-10-19T20:37:52.957502201Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T20:37:52.960420693Z","region_metadata_checked_at":"2026-10-19T20:37:52.960420693Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c6d7b4094ba7696d636f0030d351023ce5d083b8706959e646babfd146b846fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112270689380601856","occurred_at":"2026-10-19T19:52:24Z","failed_at":"provision","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T20:14:27.366Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T20:14:27.872Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T20:14:27.872Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T20:14:28.027Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-10-19T20:14:28.232Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-10-19T20:14:28.232Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T20:14:28.292Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T20:15:24.446Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T20:15:24.446Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112270564830744576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7548,"pr_state":"open","pr_sha":"4d8351df7556ab6d90240cd3820ed0ef50fa5144","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-19T19:52:01Z","started_at":"2026-10-19T19:52:01Z","completed_at":"2026-10-19T22:04:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T22:12:19.362620404Z","timing_metadata_checked_at":"2026-10-19T22:12:19.362620404Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T22:12:19.366033894Z","region_metadata_checked_at":"2026-10-19T22:12:19.366033894Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112266775457435648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7614,"pr_state":"closed","pr_sha":"7432bf500743b7d4cbed05fd3b189362bd884638","final_merged_sha":"6923433659b9bed178c1697bcc5c5ab221c339dc","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T19:36:57Z","started_at":"2026-10-19T19:36:57Z","completed_at":"2026-10-19T21:38:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:51:26.075901491Z","timing_metadata_checked_at":"2026-10-19T21:51:26.075901491Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T21:51:26.090526046Z","region_metadata_checked_at":"2026-10-19T21:51:26.090526046Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"77f249e24ffee46a3cd612866c9915f7156d747ff3e6f9c968bd9942ef5db79c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112266775457435648","occurred_at":"2026-10-19T19:36:57Z","failed_at":"e2e","signature_id":"bdbb32bcebb5a63a8a31b16a6425f1816af2a8db1eb1c49c5c7f8669296625b2","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:437]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x340bb99d4c0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-rhzhzsd2dvs2\" to finish creating: GET https://rp.j7435648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\",\n \"name\": \"1520252c-b285-49f1-8706-c82f9e3a45de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T20:28:35.166007052Z\",\n \"endTime\": \"2026-10-19T20:47:38.665697527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine s5p6y5u2k0u9n5z-worker-dlpqz-bfjsj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine s5p6y5u2k0u9n5z-worker-dlpqz-bfjsj: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7435648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\n correlation ID: da09f0ee-1b1c-4dd0-986e-72ed6522e248\n client request ID: 773c74d7-3374-4a99-9619-c2776f1df3e0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:437]: failed to create worker nodepool for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0x340bb99d4c0\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-rhzhzsd2dvs2\" to finish creating: GET https://rp.j7435648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\",\n \"name\": \"1520252c-b285-49f1-8706-c82f9e3a45de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T20:28:35.166007052Z\",\n \"endTime\": \"2026-10-19T20:47:38.665697527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine s5p6y5u2k0u9n5z-worker-dlpqz-bfjsj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine s5p6y5u2k0u9n5z-worker-dlpqz-bfjsj: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7435648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1520252c-b285-49f1-8706-c82f9e3a45de\n correlation ID: da09f0ee-1b1c-4dd0-986e-72ed6522e248\n client request ID: 773c74d7-3374-4a99-9619-c2776f1df3e0\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7a358363367e66cfa000f24735e681014fc6da77e0080ccc5f27229ff9e3bce1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112266775457435648","occurred_at":"2026-10-19T19:36:57Z","failed_at":"e2e","signature_id":"eeda0a68876291ebbf928a7c660b039924edb75924c1c795e3d5cba0f09af203","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x13c01da76240\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x13c01e312060\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13c01f5045e0\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13c01e3822a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x13c01da76240\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x13c01e312060\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13c01f5045e0\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13c01e3822a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112264983445245952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"closed","pr_sha":"7f136ea439f0e69b9cacf1a3a044164c9c5ac85f","final_merged_sha":"7f136ea439f0e69b9cacf1a3a044164c9c5ac85f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T19:29:50Z","started_at":"2026-10-19T19:29:50Z","completed_at":"2026-10-19T21:41:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:51:25.53671607Z","timing_metadata_checked_at":"2026-10-19T21:51:25.53671607Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T21:51:25.541004757Z","region_metadata_checked_at":"2026-10-19T21:51:25.541004757Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260189582266368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6866,"pr_state":"open","pr_sha":"d3c31a822d8049904889e6efe135f041de1e5ab9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-19T19:10:47Z","started_at":"2026-10-19T19:10:47Z","completed_at":"2026-10-19T21:17:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:30:42.96970453Z","timing_metadata_checked_at":"2026-10-19T21:30:42.96970453Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T21:30:42.972597021Z","region_metadata_checked_at":"2026-10-19T21:30:42.972597021Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260184683319296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7556,"pr_state":"closed","pr_sha":"098b7e743ce49a5b70c3d964d3cdd247a796ee1d","final_merged_sha":"098b7e743ce49a5b70c3d964d3cdd247a796ee1d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T19:10:46Z","started_at":"2026-10-19T19:10:46Z","completed_at":"2026-10-19T21:23:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:30:44.031731781Z","timing_metadata_checked_at":"2026-10-19T21:30:44.031731781Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T21:30:44.034532772Z","region_metadata_checked_at":"2026-10-19T21:30:44.034532772Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260172159127552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"1767bdfde51a636bd7588c375d91adc3d907bfe3","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T19:10:43Z","started_at":"2026-10-19T19:10:43Z","completed_at":"2026-10-19T19:48:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:55:32.719179304Z","timing_metadata_checked_at":"2026-10-19T19:55:32.719179304Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:55:32.722067995Z","region_metadata_checked_at":"2026-10-19T19:55:32.722067995Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"15020be11e742dc938c2ca1fc1d4c9a35e17978207cc5f8ac18c9a5829a796b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260172159127552","occurred_at":"2026-10-19T19:10:43Z","failed_at":"provision","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T19:34:54.564Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T19:34:54.990Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T19:34:54.990Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T19:34:55.299Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-19T19:34:55.456Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-19T19:34:55.456Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T19:34:55.507Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T19:35:45.081Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T19:35:45.081Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260148863963136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7497,"pr_state":"open","pr_sha":"793a8fb63e193e71a604f2de4aca38eabbab8dc9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T19:10:37Z","started_at":"2026-10-19T19:10:37Z","completed_at":"2026-10-19T21:20:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:30:43.700847424Z","timing_metadata_checked_at":"2026-10-19T21:30:43.700847424Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T21:30:43.703992114Z","region_metadata_checked_at":"2026-10-19T21:30:43.703992114Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"7dc601b92d707b7ca18f0af617aaae023c59c390a840937d7700eba9762440a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260148863963136","occurred_at":"2026-10-19T19:10:37Z","failed_at":"e2e","signature_id":"6271fa840ee018e827576ac0131944df99fffa66b27dc5c1c9b00661db8039a2","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x424386240f0\u003e: \n resource group \"delayed-rbac-9xs9kgbdmnwv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-9xs9kgbdmnwv\" to finish deleting: GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\",\n \"name\": \"2dfe8bfe-9c49-4dad-9e91-a524833d6d27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:53:00.471686176Z\",\n \"endTime\": \"2026-10-19T20:17:09.335861904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\n correlation ID: e5bf541d-883b-4a45-98e4-199dc9b858a9\n client request ID: 39529ecb-f70a-4c21-a2de-abd0e5ac1e6b\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x424386240f0\u003e: \n resource group \"delayed-rbac-9xs9kgbdmnwv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-9xs9kgbdmnwv\" to finish deleting: GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\",\n \"name\": \"2dfe8bfe-9c49-4dad-9e91-a524833d6d27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:53:00.471686176Z\",\n \"endTime\": \"2026-10-19T20:17:09.335861904Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2dfe8bfe-9c49-4dad-9e91-a524833d6d27\n correlation ID: e5bf541d-883b-4a45-98e4-199dc9b858a9\n client request ID: 39529ecb-f70a-4c21-a2de-abd0e5ac1e6b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c38f743db47d6f9f5121d516e9afceb3f75a48515944154ee4f76d3ccc9e9bfe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112260148863963136","occurred_at":"2026-10-19T19:10:37Z","failed_at":"e2e","signature_id":"b4ca3862e521c03d799aa9ddf1252fcdf206a45876ae63817e25772e37f22022","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3d4749b3c078\u003e: \n resource group \"customer-rg-j9j4x64vc846\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-j9j4x64vc846\" to finish deleting: GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\",\n \"name\": \"060bb367-1bd8-42f9-b696-549f7d9fb867\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T20:02:54.979353355Z\",\n \"endTime\": \"2026-10-19T20:27:00.270376787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ti2ch5ekujd757n62ntgfdpp1p6fpca still exists (deletion dispatched at 2026-10-19T20:02:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\n correlation ID: 1b181c6c-5659-46ac-a08f-39fecbba1c31\n client request ID: 190e259b-25d3-414f-bd78-f91c53d9c681\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3d4749b3c078\u003e: \n resource group \"customer-rg-j9j4x64vc846\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-j9j4x64vc846\" to finish deleting: GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\",\n \"name\": \"060bb367-1bd8-42f9-b696-549f7d9fb867\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T20:02:54.979353355Z\",\n \"endTime\": \"2026-10-19T20:27:00.270376787Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ti2ch5ekujd757n62ntgfdpp1p6fpca still exists (deletion dispatched at 2026-10-19T20:02:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3963136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/060bb367-1bd8-42f9-b696-549f7d9fb867\n correlation ID: 1b181c6c-5659-46ac-a08f-39fecbba1c31\n client request ID: 190e259b-25d3-414f-bd78-f91c53d9c681\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112259193590255616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"closed","pr_sha":"782bf027a9e6c9936969566aac513a7d663d5777","final_merged_sha":"45037000851deb39176b3b62f286f06464c87985","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T19:06:50Z","started_at":"2026-10-19T19:06:50Z","completed_at":"2026-10-19T19:49:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:55:32.695689775Z","timing_metadata_checked_at":"2026-10-19T19:55:32.695689775Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:55:32.700217461Z","region_metadata_checked_at":"2026-10-19T19:55:32.700217461Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"05bf44f113878f1ebf9d83f3e73c128126a56c79190326b7f8c91cca06544da3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112259193590255616","occurred_at":"2026-10-19T19:06:50Z","failed_at":"provision","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T19:36:46.694Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-19T19:36:47.222Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-19T19:36:47.222Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T19:36:47.440Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-10-19T19:36:47.554Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-10-19T19:36:47.554Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T19:36:47.600Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T19:42:29.171Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T19:42:29.171Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112258228648677376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7602,"pr_state":"closed","pr_sha":"480592cf094dfe5a02ccdacd0bc1ae203134ac73","final_merged_sha":"4c1deb264ba30a91ffe18fcc860eb45faa86f563","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T19:03:00Z","started_at":"2026-10-19T19:03:00Z","completed_at":"2026-10-19T20:58:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:09:59.318418398Z","timing_metadata_checked_at":"2026-10-19T21:09:59.318418398Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T21:09:59.322370787Z","region_metadata_checked_at":"2026-10-19T21:09:59.322370787Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"ac9bc300915377190d6ca2e6ab46f7cf18aec37b8379ddf48929f890a4f4d0e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112258228648677376","occurred_at":"2026-10-19T19:03:00Z","failed_at":"e2e","signature_id":"53bc6aee73b2dd32359da6c26960a91c8d89c84b41fd03eac07099db143926b5","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28db69b98ae0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-f6gwn5qzckpq\" to finish creating: GET https://rp.j8677376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\",\n \"name\": \"55b2a8bb-3d6c-4170-81ce-f8813840550c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:48:10.999588854Z\",\n \"endTime\": \"2026-10-19T20:07:15.605638501Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-wmvjh-tdsrw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-f6gwn5qzckpq--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-wmvjh-tdsrw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-f6gwn5qzckpq--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8677376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\n correlation ID: 43d991c8-f99d-4e5d-8930-6d417ef255cd\n client request ID: 24277a58-4912-4429-b89c-c6baeb4435fc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28db69b98ae0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-f6gwn5qzckpq\" to finish creating: GET https://rp.j8677376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\",\n \"name\": \"55b2a8bb-3d6c-4170-81ce-f8813840550c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:48:10.999588854Z\",\n \"endTime\": \"2026-10-19T20:07:15.605638501Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-wmvjh-tdsrw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-f6gwn5qzckpq--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-wmvjh-tdsrw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-f6gwn5qzckpq--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8677376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/55b2a8bb-3d6c-4170-81ce-f8813840550c\n correlation ID: 43d991c8-f99d-4e5d-8930-6d417ef255cd\n client request ID: 24277a58-4912-4429-b89c-c6baeb4435fc\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"362976a3de1c8de9ef8affd7b179e4e636ba9b64bddffacea53a745a204432e5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-wmvjh-tdsrw to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bd84355e4f131b57b2f284d0459901e3caa5a80f361c879786b6fe66678e8d72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112258228648677376","occurred_at":"2026-10-19T19:03:00Z","failed_at":"alert","signature_id":"08981d454c0029f1ada7b11e8d8104fb622c4bfeb70379b2cd58fc966c5e5ded","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s) (1 unknown, 2 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T19:57:21Z\n Ended: 2026-10-19T20:23:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8677376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti26r8lt041sjsk1mfsgims5nh4gjni-j2j6u5m5g1b2x8s\", pod=\"azure-disk-csi-driver-controller-55fb85cb4b-zqvs9\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti26r8lt041sjsk1mfsgims5nh4gjni-j2j6u5m5g1b2x8s/azure-disk-csi-driver-controller-55fb85cb4b-zqvs9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["362976a3de1c8de9ef8affd7b179e4e636ba9b64bddffacea53a745a204432e5","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112256887872294912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"closed","pr_sha":"8849e5d7c268e9779b73b664bf5de466df0d0ef6","final_merged_sha":"8849e5d7c268e9779b73b664bf5de466df0d0ef6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T18:57:40Z","started_at":"2026-10-19T18:57:40Z","completed_at":"2026-10-19T21:21:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:30:43.325598708Z","timing_metadata_checked_at":"2026-10-19T21:30:43.325598708Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T21:30:43.328664398Z","region_metadata_checked_at":"2026-10-19T21:30:43.328664398Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112256614017798144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"6083f7bc2fc85a9b131c65a81fe6d689357564ce","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-19T18:56:35Z","started_at":"2026-10-19T18:56:35Z","completed_at":"2026-10-19T21:23:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T21:30:42.671590671Z","timing_metadata_checked_at":"2026-10-19T21:30:42.671590671Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T21:30:42.675246159Z","region_metadata_checked_at":"2026-10-19T21:30:42.675246159Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112247946668937216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7602,"pr_state":"closed","pr_sha":"480592cf094dfe5a02ccdacd0bc1ae203134ac73","final_merged_sha":"4c1deb264ba30a91ffe18fcc860eb45faa86f563","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T18:22:08Z","started_at":"2026-10-19T18:22:08Z","completed_at":"2026-10-19T18:58:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:13:37.212991399Z","timing_metadata_checked_at":"2026-10-19T19:13:37.212991399Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:13:37.216528788Z","region_metadata_checked_at":"2026-10-19T19:13:37.216528788Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"78885f4c8109cdaf46180cbcca02a36cc0fc0d732500240f0b4d742febcdda12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112247946668937216","occurred_at":"2026-10-19T18:22:08Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T18:44:25.577Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-19T18:44:27.375Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-19T18:44:29.162Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=6167d0252600ec817c994890bab60d9173f655840adcabf122067da893127d1f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8937216-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6167d0252600ec817c994890bab60d9173f655840adcabf122067da893127d1f\ntime=2026-10-19T18:53:01.241Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8937216-svc/providers/Microsoft.Resources/deployments/6167d0252600ec817c994890bab60d9173f655840adcabf122067da893127d1f/operationStatuses/08584091714172679540\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8937216-svc/providers/Microsoft.Resources/deployments/profile-w5ahaumc2w6ei\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8937216-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j8937216-svc/resourceAssociations/nsp-j8937216-svc-ra-d5a635c9-43bf-5de7-aaf2-5232b9ebe5d0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8937216-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j8937216-svc/resourceAssociations/nsp-j8937216-svc-ra-d5a635c9-43bf-5de7-aaf2-5232b9ebe5d0, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112246631993708544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7616,"pr_state":"closed","pr_sha":"859f0b6dc285b366737a205f6106636453a76fa3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T18:16:55Z","started_at":"2026-10-19T18:16:55Z","completed_at":"2026-10-19T20:35:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T20:48:37.437588562Z","timing_metadata_checked_at":"2026-10-19T20:48:37.437588562Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T20:48:37.442099447Z","region_metadata_checked_at":"2026-10-19T20:48:37.442099447Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a34e31c7b61a6c749ed40d795f85b86bd198427faa9cc5cf356b5235a497c40d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112246631993708544","occurred_at":"2026-10-19T18:16:55Z","failed_at":"e2e","signature_id":"aaacb1de33b5fad81d04fae31a9da3f8a348ac3d9a71a0bc7e77be76da64316b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3895fbd87c80\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-89sq9p7645tk\" to finish creating: GET https://rp.j3708544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\",\n \"name\": \"111cac39-15a9-4a55-ab81-242565fc5fd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:38:52.630267131Z\",\n \"endTime\": \"2026-10-19T19:57:56.018733569Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj to exist\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj to exist\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3708544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\n correlation ID: 439e72b2-9c87-4ffb-ac85-0abf45d7f0d2\n client request ID: 07db40a5-8bb3-4389-ac48-a0125b515b0e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3895fbd87c80\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-89sq9p7645tk\" to finish creating: GET https://rp.j3708544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\",\n \"name\": \"111cac39-15a9-4a55-ab81-242565fc5fd7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T19:38:52.630267131Z\",\n \"endTime\": \"2026-10-19T19:57:56.018733569Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj to exist\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-87vjj to exist\\nMachine b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-89sq9p7645tk--managed/providers/Microsoft.Compute/virtualMachines/b4k7t7k1b3t7l6w-autoscale-noaz-d4kmh-zqcrh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3708544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/111cac39-15a9-4a55-ab81-242565fc5fd7\n correlation ID: 439e72b2-9c87-4ffb-ac85-0abf45d7f0d2\n client request ID: 07db40a5-8bb3-4389-ac48-a0125b515b0e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ef1a859d4a5b48ed2f975faa7a87f38129b89e34f695186a2b0e35ba0b81d5df","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ef1a859d4a5b48ed2f975faa7a87f38129b89e34f695186a2b0e35ba0b81d5df"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112244680858013696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7132,"pr_state":"closed","pr_sha":"9f60977a89121124186b3a9255a54b3e5c0b283b","final_merged_sha":"9f60977a89121124186b3a9255a54b3e5c0b283b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T18:09:10Z","started_at":"2026-10-19T18:09:10Z","completed_at":"2026-10-19T20:28:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T20:37:52.825079862Z","timing_metadata_checked_at":"2026-10-19T20:37:52.825079862Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T20:37:52.828505553Z","region_metadata_checked_at":"2026-10-19T20:37:52.828505553Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112244635597279232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7609,"pr_state":"closed","pr_sha":"2f5ec471013bfa762d22c0066746cc060390b69c","final_merged_sha":"2f5ec471013bfa762d22c0066746cc060390b69c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T18:08:59Z","started_at":"2026-10-19T18:08:59Z","completed_at":"2026-10-19T20:06:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T20:17:30.419273584Z","timing_metadata_checked_at":"2026-10-19T20:17:30.419273584Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T20:17:30.422693673Z","region_metadata_checked_at":"2026-10-19T20:17:30.422693673Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"01b1ba69e24aa503a47231edec9aae47784f1eae1c5046316ef6d76ef664e5e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112244635597279232","occurred_at":"2026-10-19T18:08:59Z","failed_at":"alert","signature_id":"49f892d173e1abb2197dc4498b86e904e0237ccafbf4467bc06a54735fd7ea43","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T18:54:11Z\n Ended: 2026-10-19T19:08:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7279232-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti1cs38lrcs57mr7od4i6kk8vg0ki0m-u4p1g2o4u2r8x4h\", pod=\"router-596877b669-ql8jm\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti1cs38lrcs57mr7od4i6kk8vg0ki0m-u4p1g2o4u2r8x4h/router-596877b669-ql8jm has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112241166756876288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"924de57c9ad6f6049da7964e3f0bebdf0cf7eb7a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T17:55:12Z","started_at":"2026-10-19T17:55:12Z","completed_at":"2026-10-19T18:18:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T18:31:13.825339831Z","timing_metadata_checked_at":"2026-10-19T18:31:13.825339831Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-19T18:31:13.828736521Z","region_metadata_checked_at":"2026-10-19T19:34:12.592832353Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ad90b5f3279eb79d7dd61820cdfa42b10060f942a63c1344a519c5c084fbfb30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112241166756876288","occurred_at":"2026-10-19T17:55:12Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112240367163150336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"1767bdfde51a636bd7588c375d91adc3d907bfe3","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T17:52:01Z","started_at":"2026-10-19T17:52:01Z","completed_at":"2026-10-19T18:26:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T18:41:33.391741332Z","timing_metadata_checked_at":"2026-10-19T18:41:33.391741332Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T18:41:33.395294322Z","region_metadata_checked_at":"2026-10-19T18:41:33.395294322Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9ac2e672a8042424053e213a9aa7f391b9cdc6810b70b90fae8f47b95a57ecff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112240367163150336","occurred_at":"2026-10-19T17:52:01Z","failed_at":"provision","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T18:14:28.628Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T18:14:29.083Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T18:14:29.083Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T18:14:29.257Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-10-19T18:14:29.460Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-10-19T18:14:29.460Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T18:14:29.517Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T18:15:22.178Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T18:15:22.178Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7566,"pr_state":"closed","pr_sha":"659b8ce018841b2a037ed9a100f8353dfc9e1551","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T17:24:53Z","started_at":"2026-10-19T17:24:53Z","completed_at":"2026-10-19T19:39:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:45:10.562326569Z","timing_metadata_checked_at":"2026-10-19T19:45:10.562326569Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:45:10.566343756Z","region_metadata_checked_at":"2026-10-19T19:45:10.566343756Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1fb79d2831bdccc4fc0651599c69e36aba5c7cee6ebf5d35d3e2eeadfa26acf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}},{"row_id":"290ee8dd7879870a342b0b25a11311b6254fb46cdd1172eb534ba52fdee0a5a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","failed_at":"e2e","signature_id":"a1239d18a63687ad99406b4ccecdd81e2142d951763a6d66f896f64e917e539e","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xdbc40b14b80\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-9ndqj45cmqg2\" to finish creating: GET https://rp.j8442752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\",\n \"name\": \"a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T18:15:40.641153712Z\",\n \"endTime\": \"2026-10-19T18:44:47.329454995Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8442752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\n correlation ID: b9b321eb-17a4-4f82-91e4-993a13652950\n client request ID: f220b5f7-1c64-40ab-bc96-b0e5ef27428d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xdbc40b14b80\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-9ndqj45cmqg2\" to finish creating: GET https://rp.j8442752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\",\n \"name\": \"a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T18:15:40.641153712Z\",\n \"endTime\": \"2026-10-19T18:44:47.329454995Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8442752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1467e3e-7ac2-4094-9be0-5ea4dcdef5e9\n correlation ID: b9b321eb-17a4-4f82-91e4-993a13652950\n client request ID: f220b5f7-1c64-40ab-bc96-b0e5ef27428d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"37ca5f236918025a51f15b5b5f48263250b15766c2ac340fc0029f7aa18ffb85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","failed_at":"alert","signature_id":"8e8af42bde7325022e45fef0ceec74e04a08f47685cabbc3285f17cf575d7f3a","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T18:34:58Z\n Ended: 2026-10-19T19:02:56Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x7lq7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T18:35:57Z\n Ended: 2026-10-19T19:06:58Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x7lq7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"742d6f643c79df82d2f8137bdb46f0ea42ab723f283d83ce12d196a703e43244","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","failed_at":"alert","signature_id":"bb91b2c59536dad8e832118575f18644aa9a309c7f2cde3ce9c9ded8d0c6d690","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T18:49:54Z\n Ended: 2026-10-19T19:02:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x7lq7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T18:52:56Z\n Ended: 2026-10-19T19:03:57Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-x7lq7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"8d916a3527a2c781ee97a55e367ef7c89cbec219c98070ce3565b23d4a3c8878","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","failed_at":"e2e","signature_id":"377d1e8a5ea363b1d3314933ad693d46789868ec6ddcce7416d4a3a08af5b8ff","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:96]: failed to create customer resources for second cluster \"basic-hcp-cluster2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cfd776cc680\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster2-vrgp2b\" in resourcegroup=\"customer-rg-d7bzdbm7gf6t\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-d7bzdbm7gf6t/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-vrgp2b/operationStatuses/08584091731103433891\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-xooij5h335npw' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-d7bzdbm7gf6t/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-vrgp2b/operationStatuses/08584091731103433891\n correlation ID: 0edba454-3af2-4d43-98da-9b281d7b5b9e\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:96]: failed to create customer resources for second cluster \"basic-hcp-cluster2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cfd776cc680\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster2-vrgp2b\" in resourcegroup=\"customer-rg-d7bzdbm7gf6t\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-d7bzdbm7gf6t/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-vrgp2b/operationStatuses/08584091731103433891\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-xooij5h335npw' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-d7bzdbm7gf6t/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-vrgp2b/operationStatuses/08584091731103433891\n correlation ID: 0edba454-3af2-4d43-98da-9b281d7b5b9e\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c782f7e7cc8c6e3a41dcf91498342b80d104e008ff520d7923d4487de91d000c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112233538538442752","occurred_at":"2026-10-19T17:24:53Z","failed_at":"alert","signature_id":"3ec9b5e5b328b5cd8a400942079982d969a6b5748d54871952c045857225f006","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T18:36:54Z\n Ended: 2026-10-19T19:06:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ab10566e-97b4-4b52-af7f-f5418105b4d1\"\n Description: Pod ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-19T18:36:54Z\n Ended: 2026-10-19T19:06:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8442752-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8442752/providers/microsoft.monitor/accounts/hcps-j8442752\", namespace=\"ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ab10566e-97b4-4b52-af7f-f5418105b4d1\"\n Description: Pod ocm-arohcpci01-2ti10j9gmvla65fehtd18pglombd8rli-y4w7z9d4n7l1q9v/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112230579016568832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7480,"pr_state":"closed","pr_sha":"440bceec3cf19e140d04f58b3a639e52101e7fa8","final_merged_sha":"230a5d27c416299f628f0667ba08deeea65c4c88","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T17:13:05Z","started_at":"2026-10-19T17:13:05Z","completed_at":"2026-10-19T19:17:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:23:56.866254929Z","timing_metadata_checked_at":"2026-10-19T19:23:56.866254929Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:23:56.869715718Z","region_metadata_checked_at":"2026-10-19T19:23:56.869715718Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9012484dc718f1f636fcbafaac443b106eedbc6a416fac144a4638eec165600b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112230579016568832","occurred_at":"2026-10-19T17:13:05Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}},{"row_id":"909b02ebf2144da29576019a134a0ca166b0ec990e47501251aefbe6453d431c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112230579016568832","occurred_at":"2026-10-19T17:13:05Z","failed_at":"alert","signature_id":"736153835c9cc9903671d0fd4f58a966daad1408a64970eb05dba83ccede2b68","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 36 time(s) (11 unknown, 25 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:30:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-w2kfh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-w2kfh has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:29:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-7s4w7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-7s4w7 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:29:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-zqd9s\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-zqd9s has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:29:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-ls5fv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-ls5fv has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:29:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-nw4vv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-nw4vv has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:29:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-bhckh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-bhckh has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:29:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-2ssnn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-2ssnn has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:29:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-dgw65\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-dgw65 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:29:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-jjkgb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-jjkgb has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:30:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-m7jnr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-m7jnr has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-19T18:05:32Z\n Ended: 2026-10-19T18:30:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6568832-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster\", pod=\"multus-admission-controller-6d89f9fc8b-dg594\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ti0ke1dfm7gh0tl4j86qi7e852u4p94-cilium-cluster/multus-admission-controller-6d89f9fc8b-dg594 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112229674405859328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6191,"pr_state":"open","pr_sha":"0b660a1c1d53873760d8044d183b30ce239eb8fb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T17:09:32Z","started_at":"2026-10-19T17:09:32Z","completed_at":"2026-10-19T19:21:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:34:12.422769957Z","timing_metadata_checked_at":"2026-10-19T19:34:12.422769957Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:34:12.426047347Z","region_metadata_checked_at":"2026-10-19T19:34:12.426047347Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6e6cf0e8c580f8212fa9ec0a7a4134b12f2efb2151a593308ba6ebe71c240eb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112229674405859328","occurred_at":"2026-10-19T17:09:32Z","failed_at":"e2e","signature_id":"820aba5d0f7b24f476521876d79e79e741d888ec783781b82502d68ebbdb843e","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39c1da55a940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-vfc45fcxtxqz\" to finish creating: GET https://rp.j5859328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\",\n \"name\": \"98bf5641-ac12-45f2-b24a-9f7eddb62757\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T18:03:22.205099398Z\",\n \"endTime\": \"2026-10-19T18:22:25.549095315Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vfc45fcxtxqz--managed/providers/Microsoft.Compute/virtualMachines/z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vfc45fcxtxqz--managed/providers/Microsoft.Compute/virtualMachines/z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5859328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\n correlation ID: 697bbb7b-4488-43cf-b80e-bef1ce56a7c0\n client request ID: 9c6f432c-bb1f-45d1-a1cc-acc4672dbd73\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39c1da55a940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-vfc45fcxtxqz\" to finish creating: GET https://rp.j5859328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\",\n \"name\": \"98bf5641-ac12-45f2-b24a-9f7eddb62757\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T18:03:22.205099398Z\",\n \"endTime\": \"2026-10-19T18:22:25.549095315Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vfc45fcxtxqz--managed/providers/Microsoft.Compute/virtualMachines/z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vfc45fcxtxqz--managed/providers/Microsoft.Compute/virtualMachines/z3r1w1e9r2b6c5r-np-1-cpcnn-wsxkm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5859328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/98bf5641-ac12-45f2-b24a-9f7eddb62757\n correlation ID: 697bbb7b-4488-43cf-b80e-bef1ce56a7c0\n client request ID: 9c6f432c-bb1f-45d1-a1cc-acc4672dbd73","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp"}},{"row_id":"bfc57da74450a3a33e484cdc02dc4ec2fa9359c1ddb535e55c0f7367b2ee471f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112229674405859328","occurred_at":"2026-10-19T17:09:32Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112228565553516544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"1767bdfde51a636bd7588c375d91adc3d907bfe3","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T17:05:07Z","started_at":"2026-10-19T17:05:07Z","completed_at":"2026-10-19T17:44:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T17:59:58.501554804Z","timing_metadata_checked_at":"2026-10-19T17:59:58.501554804Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T17:59:58.504184796Z","region_metadata_checked_at":"2026-10-19T17:59:58.504184796Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a23af3b3d5a61da5082bb815c041ce14677dd69d5c28304071c023b96beb8621","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112228565553516544","occurred_at":"2026-10-19T17:05:07Z","failed_at":"provision","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T17:29:06.855Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T17:29:07.509Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T17:29:07.509Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T17:29:07.746Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-19T17:29:08.064Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-19T17:29:08.064Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T17:29:08.191Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T17:30:30.280Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T17:30:30.280Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112222204212023296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7497,"pr_state":"open","pr_sha":"793a8fb63e193e71a604f2de4aca38eabbab8dc9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T16:39:51Z","started_at":"2026-10-19T16:39:51Z","completed_at":"2026-10-19T19:03:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:13:37.085364795Z","timing_metadata_checked_at":"2026-10-19T19:13:37.085364795Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:13:37.089833682Z","region_metadata_checked_at":"2026-10-19T19:13:37.089833682Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4ec3aa03b2326b52c749077fa29c46a6e007e90856b29a9f3d92d03b566894e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112222204212023296","occurred_at":"2026-10-19T16:39:51Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112222016026185728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7604,"pr_state":"closed","pr_sha":"35c1994a708e09591727ec4f56c0dbf1f09e0870","final_merged_sha":"35c1994a708e09591727ec4f56c0dbf1f09e0870","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T16:39:06Z","started_at":"2026-10-19T16:39:06Z","completed_at":"2026-10-19T19:17:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:23:57.347811767Z","timing_metadata_checked_at":"2026-10-19T19:23:57.347811767Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:23:57.350711058Z","region_metadata_checked_at":"2026-10-19T19:23:57.350711058Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a28ea7f8dd1b1cd01c4950f70167d3c13854077af1f00741a89f194af3acdf21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112222016026185728","occurred_at":"2026-10-19T16:39:06Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112221900917706752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7556,"pr_state":"closed","pr_sha":"098b7e743ce49a5b70c3d964d3cdd247a796ee1d","final_merged_sha":"098b7e743ce49a5b70c3d964d3cdd247a796ee1d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T16:38:38Z","started_at":"2026-10-19T16:38:38Z","completed_at":"2026-10-19T18:51:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T19:02:56.726171351Z","timing_metadata_checked_at":"2026-10-19T19:02:56.726171351Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T19:02:56.729547839Z","region_metadata_checked_at":"2026-10-19T19:02:56.729547839Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ed6d478f4622086604c51596f4c15615fd94f8cfd779338dd51f121406cc6719","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112221900917706752","occurred_at":"2026-10-19T16:38:38Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112221482724626432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7592,"pr_state":"closed","pr_sha":"ec8367fdee266ef7cd77f2342312c9c6d05c4f90","final_merged_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T16:36:57Z","started_at":"2026-10-19T16:36:57Z","completed_at":"2026-10-19T18:41:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T18:52:05.532983969Z","timing_metadata_checked_at":"2026-10-19T18:52:05.532983969Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T18:52:05.536684156Z","region_metadata_checked_at":"2026-10-19T18:52:05.536684156Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c878ad20e998bb18008b635b23d5a09892a5c5ae0dd54ac253fb4d3e60c96fa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112221482724626432","occurred_at":"2026-10-19T16:36:57Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112213551379648512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7608,"pr_state":"closed","pr_sha":"13fc7a46ccf34167569d64e82bf055c24b5fc453","final_merged_sha":"f8135fd806c5ededeeff856816d373186933c603","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T16:05:17Z","started_at":"2026-10-19T16:05:17Z","completed_at":"2026-10-19T16:57:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T17:07:16.125457401Z","timing_metadata_checked_at":"2026-10-19T17:07:16.125457401Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T17:07:16.129909687Z","region_metadata_checked_at":"2026-10-19T17:07:16.129909687Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0171f67ced040f38815a7ba8c00c7281375968d1fbaa28ed13a97ef43821a2ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112213551379648512","occurred_at":"2026-10-19T16:05:17Z","failed_at":"provision","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T16:50:11.764Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T16:50:12.180Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T16:50:12.180Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T16:50:12.348Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-10-19T16:50:12.613Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-10-19T16:50:12.613Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T16:50:12.683Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T16:51:02.039Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T16:51:02.040Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112212933982294016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7613,"pr_state":"closed","pr_sha":"e61caeed32e6f99bc7e1941b1a140f53592a5b22","final_merged_sha":"1ffb0dcbcc764e62c843aaed5a91e3d2ef0b37df","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T16:02:58Z","started_at":"2026-10-19T16:02:58Z","completed_at":"2026-10-19T18:28:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T18:41:33.545864082Z","timing_metadata_checked_at":"2026-10-19T18:41:33.545864082Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T18:41:33.549560372Z","region_metadata_checked_at":"2026-10-19T18:41:33.549560372Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"94f46dd20fa63edceed933865d161c12b04700e272127fa2cbe2aab827af3922","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112212933982294016","occurred_at":"2026-10-19T16:02:58Z","failed_at":"e2e","signature_id":"eb441e563244116bfa29f1b8638f19b4aa62c4d3fab18f60eecf424b95ccceca","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d83e42499e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-85n46zxr29bc\" to finish creating: GET https://rp.j2294016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\",\n \"name\": \"eb476919-259d-4581-ba0c-41803ac281da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T17:15:51.295694688Z\",\n \"endTime\": \"2026-10-19T17:34:52.47411646Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2294016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\n correlation ID: 2736a12e-0c94-445a-972d-f60f128898b0\n client request ID: 8ae6dbe5-3e36-4ee4-b803-b4d4643b03d4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d83e42499e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-85n46zxr29bc\" to finish creating: GET https://rp.j2294016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\",\n \"name\": \"eb476919-259d-4581-ba0c-41803ac281da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T17:15:51.295694688Z\",\n \"endTime\": \"2026-10-19T17:34:52.47411646Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2294016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/eb476919-259d-4581-ba0c-41803ac281da\n correlation ID: 2736a12e-0c94-445a-972d-f60f128898b0\n client request ID: 8ae6dbe5-3e36-4ee4-b803-b4d4643b03d4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"be07ba3e4a75962519a91ee0afe24837dd6abd1b56dc245e661b283e7211abcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112212933982294016","occurred_at":"2026-10-19T16:02:58Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112211956654936064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6866,"pr_state":"open","pr_sha":"d3c31a822d8049904889e6efe135f041de1e5ab9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T15:59:08Z","started_at":"2026-10-19T15:59:08Z","completed_at":"2026-10-19T18:37:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T18:52:05.137037793Z","timing_metadata_checked_at":"2026-10-19T18:52:05.137037793Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T18:52:05.140709481Z","region_metadata_checked_at":"2026-10-19T18:52:05.140709481Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2595b9291713353f3ed82df219e6418a8931b1129db0f1070d7d2eb27089205a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112211956654936064","occurred_at":"2026-10-19T15:59:08Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7601,"pr_state":"closed","pr_sha":"fd7a4084e07a0c7638492dd265960399c00ce236","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T15:55:16Z","started_at":"2026-10-19T15:55:16Z","completed_at":"2026-10-19T18:16:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T18:31:13.989636803Z","timing_metadata_checked_at":"2026-10-19T18:31:13.989636803Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T18:31:13.992596194Z","region_metadata_checked_at":"2026-10-19T18:31:13.992596194Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"9a7f465ad9759dbdfe589028e1202a31e123fbdba6504fc755a346647292756f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","occurred_at":"2026-10-19T15:55:16Z","failed_at":"alert","signature_id":"9c82a77768c5119269831d8f975426ffe0af91bf48ce234e95f2caa15a644081","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T17:14:26Z\n Ended: 2026-10-19T17:38:23Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0032768-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032768/providers/microsoft.monitor/accounts/hcps-j0032768\", namespace=\"ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8l75h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T17:26:26Z\n Ended: 2026-10-19T17:38:23Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0032768-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032768/providers/microsoft.monitor/accounts/hcps-j0032768\", namespace=\"ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-8l75h\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"a33614372709681b265319a3eb9d4ed86c3619d302ed5be17e9457ba8cc3b5e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","occurred_at":"2026-10-19T15:55:16Z","failed_at":"e2e","signature_id":"2f9ef72fed1ff555c11644ab75f32dedc822bf7be21ab5ff05bf932afe387496","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-bbzfqrxldsr2/cluster-candidate-5-1-vz4k7j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x399878fce5c0\u003e: \n failed to create HCP cluster cluster-candidate-5-1-vz4k7j: failed waiting for cluster=\"cluster-candidate-5-1-vz4k7j\" in resourcegroup=\"rg-candidate-5-1-bbzfqrxldsr2\" to finish creating: GET https://rp.j0032768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\",\n \"name\": \"f0513279-bd34-4b93-937c-26fa6b8a6aeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:52:45.887911577Z\",\n \"endTime\": \"2026-10-19T17:21:46.629500013Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0032768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\n correlation ID: 4cf69c21-e6dd-4c6d-8bd7-3644b7285946\n client request ID: 2f22a1ad-d176-4f31-9493-27c1ef58e861\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-1-bbzfqrxldsr2/cluster-candidate-5-1-vz4k7j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x399878fce5c0\u003e: \n failed to create HCP cluster cluster-candidate-5-1-vz4k7j: failed waiting for cluster=\"cluster-candidate-5-1-vz4k7j\" in resourcegroup=\"rg-candidate-5-1-bbzfqrxldsr2\" to finish creating: GET https://rp.j0032768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\",\n \"name\": \"f0513279-bd34-4b93-937c-26fa6b8a6aeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:52:45.887911577Z\",\n \"endTime\": \"2026-10-19T17:21:46.629500013Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0032768.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0513279-bd34-4b93-937c-26fa6b8a6aeb\n correlation ID: 4cf69c21-e6dd-4c6d-8bd7-3644b7285946\n client request ID: 2f22a1ad-d176-4f31-9493-27c1ef58e861\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bb80336f47aec8de292dc3a74765e04b4aba206f4f00813c798af29d509d9f28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","occurred_at":"2026-10-19T15:55:16Z","failed_at":"alert","signature_id":"aac13e9e48b4294d601642f38cf4f1698fe3e6c046c28a9a2ba8ffbab5af5f4a","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T17:17:20Z\n Ended: 2026-10-19T17:42:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0032768-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032768/providers/microsoft.monitor/accounts/hcps-j0032768\", namespace=\"ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"001c2a8a-fc34-4e11-96e9-d8a9bf0c0d4a\"\n Description: Pod ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-19T17:17:20Z\n Ended: 2026-10-19T17:42:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0032768-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.136:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032768/providers/microsoft.monitor/accounts/hcps-j0032768\", namespace=\"ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"001c2a8a-fc34-4e11-96e9-d8a9bf0c0d4a\"\n Description: Pod ocm-arohcpci01-2thvpnjf9f40p719moscorcvlmrkokr9-e5z0m1p8i3n8y2p/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"c539c640fb0c8e9f2a16a7747ac2982c3a507cf290ec9da4d6370797fc688960","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112210984000032768","occurred_at":"2026-10-19T15:55:16Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112207281033580544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"1767bdfde51a636bd7588c375d91adc3d907bfe3","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T15:40:33Z","started_at":"2026-10-19T15:40:33Z","completed_at":"2026-10-19T16:08:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T16:24:30.888385501Z","timing_metadata_checked_at":"2026-10-19T16:24:30.888385501Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T16:24:30.891302691Z","region_metadata_checked_at":"2026-10-19T16:24:30.891302691Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a7cb931436a2fb79e4342b8bb33a3d6859000cbd7cbb575fbedba17117eb36a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112207281033580544","occurred_at":"2026-10-19T15:40:33Z","failed_at":"provision","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T16:02:01.049Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T16:02:01.490Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T16:02:01.490Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T16:02:01.775Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-19T16:02:01.979Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-19T16:02:01.979Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T16:02:02.044Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T16:02:57.648Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T16:02:57.648Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"closed","pr_sha":"1b4e467e5aa6dc3a3f2b9a091369906bed6d2f1f","final_merged_sha":"7f136ea439f0e69b9cacf1a3a044164c9c5ac85f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T15:37:16Z","started_at":"2026-10-19T15:37:16Z","completed_at":"2026-10-19T17:57:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T18:10:28.643820754Z","timing_metadata_checked_at":"2026-10-19T18:10:28.643820754Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T18:10:28.651865429Z","region_metadata_checked_at":"2026-10-19T18:10:28.651865429Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1a3ef5d3ac3526d74c29212f02d396572eb80d7bf7f4302a6269e5084fa93bab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","failed_at":"e2e","signature_id":"0217128493b7828f90a93261926d540dcdbdc6f3405b87a78fddf4c7104ab055","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x2ba9acdcc318\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-pp8lljnf5t7j\" to finish creating: GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\",\n \"name\": \"025c0379-fa3b-4e93-b19d-0ad03edccdd2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:45:45.996504863Z\",\n \"endTime\": \"2026-10-19T17:04:48.768522414Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-7v7tf-6wgpp: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-pp8lljnf5t7j--managed/2thvfqe1vdauvp1vi8ns5oibmpnkopfg_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-pp8lljnf5t7j--managed/providers/Microsoft.Compute/availabilitySets/2thvfqe1vdauvp1vi8ns5oibmpnkopfg_cilium-cl-cilium-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '1' seconds after ad... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\n correlation ID: c38c42ba-d929-4393-9a38-4d460e6cc1c1\n client request ID: 991ecdc6-7a24-4e68-a023-12a15de7fc26\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x2ba9acdcc318\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-pp8lljnf5t7j\" to finish creating: GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\",\n \"name\": \"025c0379-fa3b-4e93-b19d-0ad03edccdd2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:45:45.996504863Z\",\n \"endTime\": \"2026-10-19T17:04:48.768522414Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-7v7tf-6wgpp: Failed: availabilitysets failed to create or update. err: failed to get existing resource cni-cilium-pp8lljnf5t7j--managed/2thvfqe1vdauvp1vi8ns5oibmpnkopfg_cilium-cl-cilium-np-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-pp8lljnf5t7j--managed/providers/Microsoft.Compute/availabilitySets/2thvfqe1vdauvp1vi8ns5oibmpnkopfg_cilium-cl-cilium-np-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '1' seconds after ad... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/025c0379-fa3b-4e93-b19d-0ad03edccdd2\n correlation ID: c38c42ba-d929-4393-9a38-4d460e6cc1c1\n client request ID: 991ecdc6-7a24-4e68-a023-12a15de7fc26\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4040a72f14f446e40df065387b843c2ab59e8482c18302866d9ef52177992739","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after ad... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"80193d79d013e1b5e751c1216665158cfdc24225d07bc98d5d462d9d15f2bd27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","failed_at":"e2e","signature_id":"aebdc6168daad45cbe0abf25f4b28797a97898ae2a5fbc5180889ca8a292b42e","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1051bd7af500\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-p7tdc2\" in resourcegroup=\"rg-np-version-upgrade-p7tdc2-lnnmsq2r4w9h\" to finish creating: GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\",\n \"name\": \"70410933-3edd-4cf6-ad9e-b8f9f2dc028a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T17:02:00.685391332Z\",\n \"endTime\": \"2026-10-19T17:21:08.269829146Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d3s0j2g6n1a2d0s-npupgrade-4-20-xl6sw-4whr7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d3s0j2g6n1a2d0s-npupgrade-4-20-xl6sw-4whr7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\n correlation ID: fb9e2f48-428d-4f30-8c61-d916aa767023\n client request ID: 93ac57ce-9b86-4ca4-8a14-339c496ee50f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1051bd7af500\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-p7tdc2\" in resourcegroup=\"rg-np-version-upgrade-p7tdc2-lnnmsq2r4w9h\" to finish creating: GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\",\n \"name\": \"70410933-3edd-4cf6-ad9e-b8f9f2dc028a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T17:02:00.685391332Z\",\n \"endTime\": \"2026-10-19T17:21:08.269829146Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d3s0j2g6n1a2d0s-npupgrade-4-20-xl6sw-4whr7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d3s0j2g6n1a2d0s-npupgrade-4-20-xl6sw-4whr7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2325504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70410933-3edd-4cf6-ad9e-b8f9f2dc028a\n correlation ID: fb9e2f48-428d-4f30-8c61-d916aa767023\n client request ID: 93ac57ce-9b86-4ca4-8a14-339c496ee50f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aefad5f94f72cb8e58bda1c7da1141a4771aeab7ed2e77e8ebf8c3e522fdd712","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","failed_at":"alert","signature_id":"8e1d1e2137e01eb5a249e9a1bb923b9498a07e23c1b6d569a52d3f1090035430","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 46 time(s) (22 unknown, 24 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T16:40:34Z\n Ended: 2026-10-19T16:56:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfqmn4vb4jp0h2nb17kirlvku8jif-r3p4f2r1s4c7j7k\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfqmn4vb4jp0h2nb17kirlvku8jif-r3p4f2r1s4c7j7k/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T16:41:37Z\n Ended: 2026-10-19T16:48:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfuaumdfma93r70770i2lk0ceibuu-l1u4a6z7l3x3m9t\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfuaumdfma93r70770i2lk0ceibuu-l1u4a6z7l3x3m9t/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T16:41:38Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfscjl15blbh8mcver1v3o3u4u0ao-y6i4i5p1q7w7n4v\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfscjl15blbh8mcver1v3o3u4u0ao-y6i4i5p1q7w7n4v/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T16:41:38Z\n Ended: 2026-10-19T16:57:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfqlloamhkfi0hhbkskh026c53tqv-x1z4l6s3s3f6f6o\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfqlloamhkfi0hhbkskh026c53tqv-x1z4l6s3s3f6f6o/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T16:42:35Z\n Ended: 2026-10-19T16:56:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfu9i3hibd7t67f8k4ikin22uvov0-ksm-hcp-metrics\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfu9i3hibd7t67f8k4ikin22uvov0-ksm-hcp-metrics/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T16:42:35Z\n Ended: 2026-10-19T16:56:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfsq2gk0ogv6igthdon43ohkqcll6-o3h6i5c2r6h8g6m\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfsq2gk0ogv6igthdon43ohkqcll6-o3h6i5c2r6h8g6m/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T16:42:35Z\n Ended: 2026-10-19T16:56:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfs4dgfss7ak7ob076ptv9s5b15c3-l9k0d8k9y9c4d6s\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfs4dgfss7ak7ob076ptv9s5b15c3-l9k0d8k9y9c4d6s/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T16:42:35Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfrp642h257s3p2mm1u1cbq24avuc-p0w0i8v2l6f3j9u\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfrp642h257s3p2mm1u1cbq24avuc-p0w0i8v2l6f3j9u/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T16:42:35Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfq8h15dkpqo3c05ofeksdhql6h0u-z7z1l9c1f6v5r1j\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfq8h15dkpqo3c05ofeksdhql6h0u-z7z1l9c1f6v5r1j/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T16:42:36Z\n Ended: 2026-10-19T16:56:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfstjgcqtr9i5gin0ic7r9c1odfdl-n3k2a6t9l8q5x7n\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfstjgcqtr9i5gin0ic7r9c1odfdl-n3k2a6t9l8q5x7n/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-19T16:42:36Z\n Ended: 2026-10-19T16:56:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfqm9keb8d2iec07muheaef524p1o-w4e1x7i1r6r0j5e\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfqm9keb8d2iec07muheaef524p1o-w4e1x7i1r6r0j5e/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-19T16:42:36Z\n Ended: 2026-10-19T16:56:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfrbnige07o90mmp5ao4p4ebo39p9-g0c7o0t4o8h3w0s\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfrbnige07o90mmp5ao4p4ebo39p9-g0c7o0t4o8h3w0s/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-19T16:42:36Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfricr8n4cfb8phakp84govl5sbp4-ea-cluster\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfricr8n4cfb8phakp84govl5sbp4-ea-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-19T16:43:35Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfs8va9j8splokkrcarauj3mssv2q-l5u5w9u0a1b4y4g\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfs8va9j8splokkrcarauj3mssv2q-l5u5w9u0a1b4y4g/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-19T16:43:35Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfrnukiqqd5c22t25k5k1vr8fpt73-j4c2r5k0m9r0j9c\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfrnukiqqd5c22t25k5k1vr8fpt73-j4c2r5k0m9r0j9c/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-19T16:43:35Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfspu00q5i9vof2vd94tgeok1kr3n-basic-cluster\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfspu00q5i9vof2vd94tgeok1kr3n-basic-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-19T16:43:35Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvftigig9mm991ouavfjlt16738njh-v0a4e1f5j5b2m0a\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvftigig9mm991ouavfjlt16738njh-v0a4e1f5j5b2m0a/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-19T16:44:35Z\n Ended: 2026-10-19T16:56:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfqju2ouj4onjdmo5mfj9dqmukjro-v20261001\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfqju2ouj4onjdmo5mfj9dqmukjro-v20261001/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-19T16:44:35Z\n Ended: 2026-10-19T16:56:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvgh1ga2vr8rfio4pgkn2vv1697h7e-cilium-cluster\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvgh1ga2vr8rfio4pgkn2vv1697h7e-cilium-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-19T16:44:35Z\n Ended: 2026-10-19T16:57:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfrkvqcg5e1rt0aettqt8df1df3pl-b4t6o0h2z3h3r6l\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfrkvqcg5e1rt0aettqt8df1df3pl-b4t6o0h2z3h3r6l/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-19T16:44:35Z\n Ended: 2026-10-19T16:58:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvfqh42qu5vrcpacs896nrie1l1pul-z1u5u6p0i7q4q4f\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvfqh42qu5vrcpacs896nrie1l1pul-z1u5u6p0i7q4q4f/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-19T16:45:35Z\n Ended: 2026-10-19T16:56:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thvgrpsbbr2rlo3i4jfsgsa1mbns4b8-agg-cluster\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thvgrpsbbr2rlo3i4jfsgsa1mbns4b8-agg-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"cf3700c1f563db499028b9b16e728791027bd3424f74694c17e1c9f7d182f292","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","failed_at":"alert","signature_id":"44fdc585ddfc610cc29e0362825162ffafcbac3e8a2962c6caf3f0426cf09c09","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T16:51:41Z\n Ended: 2026-10-19T16:57:41Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2325504/providers/microsoft.monitor/accounts/hcps-j2325504\", namespace=\"ocm-arohcpci01-2thvfscjl15blbh8mcver1v3o3u4u0ao-y6i4i5p1q7w7n4v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2wfh7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thvfscjl15blbh8mcver1v3o3u4u0ao-y6i4i5p1q7w7n4v/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T16:51:41Z\n Ended: 2026-10-19T16:57:41Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2325504/providers/microsoft.monitor/accounts/hcps-j2325504\", namespace=\"ocm-arohcpci01-2thvfqlloamhkfi0hhbkskh026c53tqv-x1z4l6s3s3f6f6o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2wfh7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thvfqlloamhkfi0hhbkskh026c53tqv-x1z4l6s3s3f6f6o/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T16:51:41Z\n Ended: 2026-10-19T16:57:42Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2325504/providers/microsoft.monitor/accounts/hcps-j2325504\", namespace=\"ocm-arohcpci01-2thvfscjl15blbh8mcver1v3o3u4u0ao-y6i4i5p1q7w7n4v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2wfh7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thvfscjl15blbh8mcver1v3o3u4u0ao-y6i4i5p1q7w7n4v/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T16:51:41Z\n Ended: 2026-10-19T16:57:41Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2325504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2325504/providers/microsoft.monitor/accounts/hcps-j2325504\", namespace=\"ocm-arohcpci01-2thvfqlloamhkfi0hhbkskh026c53tqv-x1z4l6s3s3f6f6o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2wfh7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thvfqlloamhkfi0hhbkskh026c53tqv-x1z4l6s3s3f6f6o/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"dcc467b33f202329339fa5a0cc74d0f117d1eac958590555fb31bbd19683cfda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112206455762325504","occurred_at":"2026-10-19T15:37:16Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["4040a72f14f446e40df065387b843c2ab59e8482c18302866d9ef52177992739","87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205178445762560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7132,"pr_state":"closed","pr_sha":"9f60977a89121124186b3a9255a54b3e5c0b283b","final_merged_sha":"9f60977a89121124186b3a9255a54b3e5c0b283b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T15:32:03Z","started_at":"2026-10-19T15:32:03Z","completed_at":"2026-10-19T17:52:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T17:59:58.480730567Z","timing_metadata_checked_at":"2026-10-19T17:59:58.480730567Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T17:59:58.484305156Z","region_metadata_checked_at":"2026-10-19T17:59:58.484305156Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"467b8315a7991db9aea8f31bfe7cd9d599e4a28391e86e6ff1846e8ce6935a0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205178445762560","occurred_at":"2026-10-19T15:32:03Z","failed_at":"e2e","signature_id":"f74d9e159d8da00a8b5ec1b1bb62fcea11dcb160c4ea6098a0c46c6cca9dfd86","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f88271a0ea0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2lvnhrzd5g7t\" to finish creating: GET https://rp.j5762560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\",\n \"name\": \"9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:38:57.011707926Z\",\n \"endTime\": \"2026-10-19T16:58:00.847122787Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine p7u4k9n2d7l9v9q-autoscale-noaz-v7npk-lb6bv: NodeDoesNotExist: Node does not exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine p7u4k9n2d7l9v9q-autoscale-noaz-v7npk-lb6bv: Deleting: virtualmachine deleting\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5762560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\n correlation ID: 78955e10-7740-4b87-a377-073a6ecddbd0\n client request ID: a8ccc39c-63f9-475d-8c01-623fcd9a0f40\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3f88271a0ea0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2lvnhrzd5g7t\" to finish creating: GET https://rp.j5762560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\",\n \"name\": \"9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:38:57.011707926Z\",\n \"endTime\": \"2026-10-19T16:58:00.847122787Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine p7u4k9n2d7l9v9q-autoscale-noaz-v7npk-lb6bv: NodeDoesNotExist: Node does not exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine p7u4k9n2d7l9v9q-autoscale-noaz-v7npk-lb6bv: Deleting: virtualmachine deleting\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5762560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9b5197c8-64c6-49b3-b12c-d01b0a5ee1e9\n correlation ID: 78955e10-7740-4b87-a377-073a6ecddbd0\n client request ID: a8ccc39c-63f9-475d-8c01-623fcd9a0f40\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb1eb44ae687b456506e286274b48afd57d492aeeab49cb4b52c4310e1e06f33","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeDoesNotExist: Node does not exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Deleting: virtualmachine deleting; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"817071a603402935ca8b53e8ad799f34e2f0023e94c7a71eaff7f83987da79e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205178445762560","occurred_at":"2026-10-19T15:32:03Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}},{"row_id":"c530ba91bfa9c49b6749ace8fb46e4c84030ffe818a161a1e6213557324ea48c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205178445762560","occurred_at":"2026-10-19T15:32:03Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T16:38:15Z\n Ended: 2026-10-19T16:53:16Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5762560-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","cb1eb44ae687b456506e286274b48afd57d492aeeab49cb4b52c4310e1e06f33"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205019058016256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7609,"pr_state":"closed","pr_sha":"2f5ec471013bfa762d22c0066746cc060390b69c","final_merged_sha":"2f5ec471013bfa762d22c0066746cc060390b69c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T15:31:34Z","started_at":"2026-10-19T15:31:34Z","completed_at":"2026-10-19T17:39:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T17:48:59.408974224Z","timing_metadata_checked_at":"2026-10-19T17:48:59.408974224Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T17:48:59.432423246Z","region_metadata_checked_at":"2026-10-19T17:48:59.432423246Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"222fde434e5dd86fb5551f64f5b6ff7d3bcd5466706f9debb336b683349a6df1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205019058016256","occurred_at":"2026-10-19T15:31:34Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}},{"row_id":"f18ec583d437437aeb074903cee1b36585843cb2ecf74b62afbe4f8a5715033f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7609/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112205019058016256","occurred_at":"2026-10-19T15:31:34Z","failed_at":"alert","signature_id":"85e0534a15b4dde124a5fdccdabf16fc4b85dce9f7c3b17f9ae34aed46461e4c","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T16:27:31Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8016256-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thv7hcqcal2f2hpopre8d6de2pirqto-v7m5p2q2k9d1x6l\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thv7hcqcal2f2hpopre8d6de2pirqto-v7m5p2q2k9d1x6l/etcd-1 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112203264580653056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"closed","pr_sha":"9b27356ba9d6d8386bb16cfda35989b545337915","final_merged_sha":"45037000851deb39176b3b62f286f06464c87985","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T15:24:35Z","started_at":"2026-10-19T15:24:35Z","completed_at":"2026-10-19T16:08:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T16:24:30.629980973Z","timing_metadata_checked_at":"2026-10-19T16:24:30.629980973Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T16:24:30.635807753Z","region_metadata_checked_at":"2026-10-19T16:24:30.635807753Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bd78d132bdaa1f9dbcf1884f086be1a77d2605426a054170a6861778d1c0aca0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112203264580653056","occurred_at":"2026-10-19T15:24:35Z","failed_at":"provision","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T15:56:31.916Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-10-19T15:56:32.339Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157]\"\ntime=2026-10-19T15:56:32.339Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T15:56:32.526Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-10-19T15:56:32.659Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-10-19T15:56:32.659Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T15:56:32.731Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-10-19T16:02:20.224Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T16:02:20.224Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199925969195008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7597,"pr_state":"closed","pr_sha":"d071a83f32db9a874ed681ae192ed18ce7bc5bf0","final_merged_sha":"322d4798e9f946e8950f4ab188bec64d82a6143a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T15:11:19Z","started_at":"2026-10-19T15:11:19Z","completed_at":"2026-10-19T17:33:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T17:48:59.265751804Z","timing_metadata_checked_at":"2026-10-19T17:48:59.265751804Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T17:48:59.272695881Z","region_metadata_checked_at":"2026-10-19T17:48:59.272695881Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3cf45eac54cb5675750f498a74a213c980180da940b930c0e968f52d2b7b45f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199925969195008","occurred_at":"2026-10-19T15:11:19Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}},{"row_id":"55aefc6f56c7be75e88905df91dd1c14ede3425b1fa942cd2eb266c1602ab343","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199925969195008","occurred_at":"2026-10-19T15:11:19Z","failed_at":"e2e","signature_id":"d79807bc63d1c36ab2b3708c01de13b2aeba0757391700b53e65ce99e6bbb469","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d6e7c8ac020\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-t27g7scwdvdr\" to finish creating: GET https://rp.j9195008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\",\n \"name\": \"04516756-3c0d-4cb8-9bea-3b83299b21af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:30:35.481364162Z\",\n \"endTime\": \"2026-10-19T16:49:44.492413303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-t27g7scwdvdr--managed/providers/Microsoft.Compute/virtualMachines/j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-t27g7scwdvdr--managed/providers/Microsoft.Compute/virtualMachines/j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9195008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\n correlation ID: 164cf44e-1a4c-43ef-ad03-502d13c22da5\n client request ID: 949d080f-05a2-40b5-b875-b9fbced5c91f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d6e7c8ac020\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-t27g7scwdvdr\" to finish creating: GET https://rp.j9195008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\",\n \"name\": \"04516756-3c0d-4cb8-9bea-3b83299b21af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T16:30:35.481364162Z\",\n \"endTime\": \"2026-10-19T16:49:44.492413303Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-t27g7scwdvdr--managed/providers/Microsoft.Compute/virtualMachines/j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-t27g7scwdvdr--managed/providers/Microsoft.Compute/virtualMachines/j9y3e8x7n5w0j9t-autoscale-noaz-wr9r9-5zvr8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9195008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04516756-3c0d-4cb8-9bea-3b83299b21af\n correlation ID: 164cf44e-1a4c-43ef-ad03-502d13c22da5\n client request ID: 949d080f-05a2-40b5-b875-b9fbced5c91f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7301,"pr_state":"open","pr_sha":"e9f8f976589a87887c08fa109716232faf23cca3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T15:09:38Z","started_at":"2026-10-19T15:09:38Z","completed_at":"2026-10-19T17:20:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T17:28:17.380504526Z","timing_metadata_checked_at":"2026-10-19T17:28:17.380504526Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T17:28:17.384847413Z","region_metadata_checked_at":"2026-10-19T17:28:17.384847413Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2447009e1844bd428c2cde00536a80b6c0d2e1a6b0bf13d1825408c309691a8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","occurred_at":"2026-10-19T15:09:38Z","failed_at":"alert","signature_id":"6c8c0b3e9ef7152e716e2cd26aff799256e0121f5ab204a6b434f09e208def1d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 92 time(s) (91 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-52xls\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-52xls has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-8pj5j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-8pj5j has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-872zw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-872zw has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-hlzns\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-hlzns has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-cz2v8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-cz2v8 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-mw5bq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-mw5bq has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-mlxzc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-mlxzc has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-zldjh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-zldjh has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-75d6b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-75d6b has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-lzmgv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-lzmgv has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-b88nc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-b88nc has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-dzd9v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-dzd9v has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-8jtmm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-8jtmm has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-kwh7b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-kwh7b has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-9b4lv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-9b4lv has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-4b762\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-4b762 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-ptzhd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-ptzhd has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-ntc8c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-ntc8c has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-c274j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-c274j has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-c4w8j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-c4w8j has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-ctjgl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-ctjgl has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-r7c52\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-r7c52 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-dpqxk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-dpqxk has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-c4lpf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-c4lpf has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-k9pxx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-k9pxx has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-79g9t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-79g9t has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-nqjtj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-nqjtj has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-w5zkh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-w5zkh has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-st8sn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-st8sn has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-p56lh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-p56lh has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-clhxc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-clhxc has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-6cgkc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-6cgkc has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-4qmk4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-4qmk4 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-psqcc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-psqcc has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-kpddb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-kpddb has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-5pkwb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-5pkwb has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-7x9vn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-7x9vn has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-bhl9l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-bhl9l has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-hmx5b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-hmx5b has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-kwrch\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-kwrch has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-whdp8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-whdp8 has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-zmr7m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-zmr7m has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-lhmw6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-lhmw6 has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-p5b9c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-p5b9c has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-zfnjh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-zfnjh has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-shf54\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-shf54 has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-k9fm5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-k9fm5 has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-tcl54\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-tcl54 has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-z84cl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-z84cl has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-k72n4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-k72n4 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-9n9rh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-9n9rh has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-zbjdg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-zbjdg has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-jdkf6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-jdkf6 has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-rtqkz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-rtqkz has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-w4sj4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-w4sj4 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-7tjw6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-7tjw6 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-stzcg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-stzcg has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-tgdzr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-tgdzr has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-svk9f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-svk9f has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-sv2t5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-sv2t5 has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-rpf9p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-rpf9p has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-qznl4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-qznl4 has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:37:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-45tq6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-45tq6 has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-h55sn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-h55sn has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-twzxj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-twzxj has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-kz995\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-kz995 has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-fv8xv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-fv8xv has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-bbz4w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-bbz4w has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-c6ss9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-c6ss9 has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-6vj24\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-6vj24 has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-qbjp9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-qbjp9 has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-7dn78\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-7dn78 has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-txj72\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-txj72 has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-mwfqs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-mwfqs has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-5bt4t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-5bt4t has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-s92xf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-s92xf has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-f5nxl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-f5nxl has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-pwdbs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-pwdbs has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:38:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-6nt9d\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-6nt9d has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-64c6c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-64c6c has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-lh56t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-lh56t has been in a non-ready state for longer than 5 minutes.\nFiring 82:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-kqdst\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-kqdst has been in a non-ready state for longer than 5 minutes.\nFiring 83:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-w824m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-w824m has been in a non-ready state for longer than 5 minutes.\nFiring 84:\n State: Resolved\n Started: 2026-10-19T16:15:27Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-d6q4r\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-d6q4r has been in a non-ready state for longer than 5 minutes.\nFiring 85:\n State: Resolved\n Started: 2026-10-19T16:15:28Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-n59nc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-n59nc has been in a non-ready state for longer than 5 minutes.\nFiring 86:\n State: Resolved\n Started: 2026-10-19T16:15:28Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-lzjfr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-lzjfr has been in a non-ready state for longer than 5 minutes.\nFiring 87:\n State: Resolved\n Started: 2026-10-19T16:15:28Z\n Ended: 2026-10-19T16:33:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-blh9z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-blh9z has been in a non-ready state for longer than 5 minutes.\nFiring 88:\n State: Resolved\n Started: 2026-10-19T16:15:28Z\n Ended: 2026-10-19T16:33:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-8lcct\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-8lcct has been in a non-ready state for longer than 5 minutes.\nFiring 89:\n State: Resolved\n Started: 2026-10-19T16:15:28Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-tgq4m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-tgq4m has been in a non-ready state for longer than 5 minutes.\nFiring 90:\n State: Resolved\n Started: 2026-10-19T16:15:28Z\n Ended: 2026-10-19T16:33:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ignition-server-proxy-5678699d94-6s756\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ignition-server-proxy-5678699d94-6s756 has been in a non-ready state for longer than 5 minutes.\nFiring 91:\n State: Resolved\n Started: 2026-10-19T16:16:31Z\n Ended: 2026-10-19T16:38:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5037184-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h\", pod=\"ovnkube-control-plane-566d46bb86-j5r7h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thuupsd8j59p35gb9lk4m4qnu0aiu8m-i5w5x8k4i5q6a4h/ovnkube-control-plane-566d46bb86-j5r7h has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"5c10ca2f880c2fcdd2640157207fb06777b7296e4616c6498154e66f1109795e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","occurred_at":"2026-10-19T15:09:38Z","failed_at":"e2e","signature_id":"88045e20d4aa89c6b9271f51a87c8e3f4bbca7bf37da9e0736de49e88fc8b858","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-sjxzmq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e0c69717cc0\u003e: \n DNS for route host agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3e0c6a7f4740\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-sjxzmq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e0c69717cc0\u003e: \n DNS for route host agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3e0c6a7f4740\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-mbxj4.apps.aro.i3o6o2t9x9d5w5r.jkfh.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"e8b3d32b71ac32789bc0539eba6581219f7c89e9c495c0bf9f5a986d6d92198e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","occurred_at":"2026-10-19T15:09:38Z","failed_at":"e2e","signature_id":"fed984ab46e663f21fa5a7bc5ab953f16af1cfb5d9265996c796fa791e841180","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0x1ffd80fcc600\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1ffd80c0ac80\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ffd811f6800\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ffd80d0e5e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0x1ffd80fcc600\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1ffd80c0ac80\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ffd811f6800\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1ffd80d0e5e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-6r5mt.apps.aro.n2r0d3y7o1g9i7q.hc5u.j5037184.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-26v8b.apps.aro.agg-cluster.4cnq.j7435648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"e993acb58a1736e72032fbc85d1ab03ae631aaa0e082174e1343cbd0148a779b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7301/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112199501795037184","occurred_at":"2026-10-19T15:09:38Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7506/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112197072718401536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7506,"pr_state":"closed","pr_sha":"221c794196b0d33ae704b959bb43e3250906e95c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T14:59:44Z","started_at":"2026-10-19T14:59:44Z","completed_at":"2026-10-19T15:53:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T16:02:41.622741662Z","timing_metadata_checked_at":"2026-10-19T16:02:41.622741662Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T16:02:41.625904553Z","region_metadata_checked_at":"2026-10-19T16:02:41.625904553Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b04740487ea95203a271e3ef77367f6159e6bc65037c5c86c9d132662de5e0bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7506/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112197072718401536","occurred_at":"2026-10-19T14:59:44Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T15:30:44.609Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-19T15:30:46.080Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8401536-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-19T15:30:46.080Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8401536-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-19T15:30:46.888Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8401536-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-19T15:30:46.889Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8401536-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-19T15:30:46.889Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8401536-svc\ntime=2026-10-19T15:45:50.026Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112195721783414784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"1767bdfde51a636bd7588c375d91adc3d907bfe3","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T14:54:36Z","started_at":"2026-10-19T14:54:36Z","completed_at":"2026-10-19T15:34:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T15:41:43.503029566Z","timing_metadata_checked_at":"2026-10-19T15:41:43.503029566Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T15:41:43.505780656Z","region_metadata_checked_at":"2026-10-19T15:41:43.505780656Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"313ce4469603e6dfd6d38dfa6c6a134a30caafcff0de8546f527e1d6c7b7d3ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112195721783414784","occurred_at":"2026-10-19T14:54:36Z","failed_at":"provision","signature_id":"19498df56e93c37fe7f6fb08ad99bc2b1b1c8d05100b0a59886217e4fe1408e0","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T15:27:02.707Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T15:27:03.206Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T15:27:03.206Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T15:27:03.631Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 namespace=default-sc\ntime=2026-10-19T15:27:03.853Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 dryRun=false\ntime=2026-10-19T15:27:03.853Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T15:27:03.915Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2\ntime=2026-10-19T15:28:05.016Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T15:28:05.016Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112193616469299200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7387,"pr_state":"closed","pr_sha":"14770cd53f0f4c113c23a8835a0dc3e9f4e40aeb","final_merged_sha":"14770cd53f0f4c113c23a8835a0dc3e9f4e40aeb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T14:46:15Z","started_at":"2026-10-19T14:46:15Z","completed_at":"2026-10-19T17:16:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T17:28:17.591266562Z","timing_metadata_checked_at":"2026-10-19T17:28:17.591266562Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T17:28:17.603966922Z","region_metadata_checked_at":"2026-10-19T17:28:17.603966922Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e20cedcc642dcf14390a3b1d6b7561b87d412664746fe993a6245cc90eb2dd5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112193616469299200","occurred_at":"2026-10-19T14:46:15Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112190987718627328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"closed","pr_sha":"21254f9bf8ade189abc24d8cb1c51f6db24d8533","final_merged_sha":"8849e5d7c268e9779b73b664bf5de466df0d0ef6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T14:35:30Z","started_at":"2026-10-19T14:35:30Z","completed_at":"2026-10-19T17:08:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T17:17:38.842781273Z","timing_metadata_checked_at":"2026-10-19T17:17:38.842781273Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T17:17:38.846743161Z","region_metadata_checked_at":"2026-10-19T17:17:38.846743161Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9f35cfb96375fd142ac11876fe281b5a3715c8aac55852994181a88cb49c4637","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112190987718627328","occurred_at":"2026-10-19T14:35:30Z","failed_at":"e2e","signature_id":"857ad3c84c83c23ff39c0a1984f03a5c20f67f7eae51b6788cb5159fcdd164ca","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b24ac0020c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-p69pvshqvq9p\" to finish creating: GET https://rp.j8627328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\",\n \"name\": \"c4f1b382-5a50-42f4-93c5-591e1ed0f369\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T15:56:13.653725725Z\",\n \"endTime\": \"2026-10-19T16:15:18.87162238Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-p69pvshqvq9p--managed/providers/Microsoft.Compute/virtualMachines/r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-p69pvshqvq9p--managed/providers/Microsoft.Compute/virtualMachines/r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8627328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\n correlation ID: 5cc9d41d-adc2-4131-802d-17d3ab484012\n client request ID: b9e380f6-f1bc-4da7-bbfb-dbb4f50cb471\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b24ac0020c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-p69pvshqvq9p\" to finish creating: GET https://rp.j8627328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\",\n \"name\": \"c4f1b382-5a50-42f4-93c5-591e1ed0f369\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T15:56:13.653725725Z\",\n \"endTime\": \"2026-10-19T16:15:18.87162238Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-p69pvshqvq9p--managed/providers/Microsoft.Compute/virtualMachines/r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-p69pvshqvq9p--managed/providers/Microsoft.Compute/virtualMachines/r0m5h3v1x6r8h3a-np-1-4x7nf-kblcl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8627328.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c4f1b382-5a50-42f4-93c5-591e1ed0f369\n correlation ID: 5cc9d41d-adc2-4131-802d-17d3ab484012\n client request ID: b9e380f6-f1bc-4da7-bbfb-dbb4f50cb471\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112186650028150784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7556,"pr_state":"closed","pr_sha":"098b7e743ce49a5b70c3d964d3cdd247a796ee1d","final_merged_sha":"098b7e743ce49a5b70c3d964d3cdd247a796ee1d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T14:18:34Z","started_at":"2026-10-19T14:18:34Z","completed_at":"2026-10-19T16:36:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T16:45:54.698109469Z","timing_metadata_checked_at":"2026-10-19T16:45:54.698109469Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T16:45:54.702111355Z","region_metadata_checked_at":"2026-10-19T16:45:54.702111355Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"20ddf62fccda689bc29c08c8a2b53d06fb4101d41ac252c558e7c8a98fa9dda5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112186650028150784","occurred_at":"2026-10-19T14:18:34Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}},{"row_id":"ea93b5f269c07663157e832b962c45a8721dfbb2aecb3e4c409a6d36f43277f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112186650028150784","occurred_at":"2026-10-19T14:18:34Z","failed_at":"alert","signature_id":"fba563d3cf21c439e21e52fbff6e8826a48f4c80cb44b801ea66fc423a3d8251","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (2 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T15:26:26Z\n Ended: 2026-10-19T15:47:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8150784-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thu6ecd4n5sg6hvpveee3vhkda08haq-j0e0i2a0u5b4n0x\", pod=\"multus-admission-controller-5d6cb76b4d-pzd4m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thu6ecd4n5sg6hvpveee3vhkda08haq-j0e0i2a0u5b4n0x/multus-admission-controller-5d6cb76b4d-pzd4m has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T15:26:26Z\n Ended: 2026-10-19T15:43:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8150784-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thu6ecd4n5sg6hvpveee3vhkda08haq-j0e0i2a0u5b4n0x\", pod=\"multus-admission-controller-5d6cb76b4d-qpjgm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thu6ecd4n5sg6hvpveee3vhkda08haq-j0e0i2a0u5b4n0x/multus-admission-controller-5d6cb76b4d-qpjgm has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112184932984950784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7480,"pr_state":"closed","pr_sha":"440bceec3cf19e140d04f58b3a639e52101e7fa8","final_merged_sha":"230a5d27c416299f628f0667ba08deeea65c4c88","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T14:11:45Z","started_at":"2026-10-19T14:11:45Z","completed_at":"2026-10-19T16:28:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T16:35:23.588575394Z","timing_metadata_checked_at":"2026-10-19T16:35:23.588575394Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T16:35:23.591423286Z","region_metadata_checked_at":"2026-10-19T16:35:23.591423286Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0791b9489e7b82bc1d773173d91425649fad13bd4a9a29c24b46c7b5befaba49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112184932984950784","occurred_at":"2026-10-19T14:11:45Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}},{"row_id":"929e11df83f2b8a4739666cb877c928903bd74f4acbb0f69d892ef13654a685d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112184932984950784","occurred_at":"2026-10-19T14:11:45Z","failed_at":"alert","signature_id":"b398e3b4b60ce0bfe92ea3c19658030c4ec5dee4a44f8a5f5c5664c590af5113","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T15:19:57Z\n Ended: 2026-10-19T15:33:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4950784-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T15:24:05Z\n Ended: 2026-10-19T15:33:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4950784-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112183526613848064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"closed","pr_sha":"6a63cbe175354d4117572e89d9aa34ce1b7ac012","final_merged_sha":"8d693aca9ae999817853c4e772be53c43796d75a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T14:06:05Z","started_at":"2026-10-19T14:06:05Z","completed_at":"2026-10-19T16:19:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T16:35:23.007333861Z","timing_metadata_checked_at":"2026-10-19T16:35:23.007333861Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T16:35:23.010901052Z","region_metadata_checked_at":"2026-10-19T16:35:23.010901052Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f6113a6adb59ce095c5330a4e1e928b78854c8e2072a7f256fe7063ac872ea3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112183526613848064","occurred_at":"2026-10-19T14:06:05Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112181488601534464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7592,"pr_state":"closed","pr_sha":"cb90a18315601e933f9406a5a586fd3762eb5303","final_merged_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T13:58:03Z","started_at":"2026-10-19T13:58:03Z","completed_at":"2026-10-19T14:52:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T14:59:27.603095157Z","timing_metadata_checked_at":"2026-10-19T14:59:27.603095157Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T14:59:27.611049031Z","region_metadata_checked_at":"2026-10-19T14:59:27.611049031Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ea17454c4457057f8c865ad9d34ceb1697f17c2bbb90e44016a9013c86065539","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112181488601534464","occurred_at":"2026-10-19T13:58:03Z","failed_at":"provision","signature_id":"6c11c44592820325fa1934724e67fb473057c5c4359853d87ec5141e3481ef08","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T14:43:12.749Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 description=\"Step nsp\\n Kind: ARM\\n Template: templates/mgmt-nsp.bicep\\n Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-10-19T14:43:13.235Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1\ntime=2026-10-19T14:43:13.684Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 deployment=299b7403a208c1c427ae8ebbcd18bb4f4fb4ff513a91ec66ec3b8e76bf4affb0 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1534464-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F299b7403a208c1c427ae8ebbcd18bb4f4fb4ff513a91ec66ec3b8e76bf4affb0\ntime=2026-10-19T14:43:43.960Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1534464-mgmt-1/providers/Microsoft.Resources/deployments/299b7403a208c1c427ae8ebbcd18bb4f4fb4ff513a91ec66ec3b8e76bf4affb0/operationStatuses/08584091858920261215\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1534464-mgmt-1/providers/Microsoft.Resources/deployments/profile-xcn2acurowiq2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1534464-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j1534464-mgmt-1/resourceAssociations/nsp-j1534464-mgmt-1-ra-66bb4858-d2d4-5bd8-8103-db26bfc7bd94\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1534464-mgmt-1/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j1534464-mgmt-1/resourceAssociations/nsp-j1534464-mgmt-1-ra-66bb4858-d2d4-5bd8-8103-db26bfc7bd94, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112179803346964480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7601,"pr_state":"closed","pr_sha":"fd7a4084e07a0c7638492dd265960399c00ce236","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T13:51:21Z","started_at":"2026-10-19T13:51:21Z","completed_at":"2026-10-19T13:52:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T13:56:51.207866153Z","timing_metadata_checked_at":"2026-10-19T13:56:51.207866153Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-19T13:56:51.210811544Z","region_metadata_checked_at":"2026-10-19T14:59:27.704554834Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"46812b8df400e62846eee61a7338cf929bc38f58757c1a8a45556d9fa92e5881","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112179803346964480","occurred_at":"2026-10-19T13:51:21Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112177636460466176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7220,"pr_state":"open","pr_sha":"540856c435b752648f8f78fe4a157f67de6eed0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T13:42:45Z","started_at":"2026-10-19T13:42:45Z","completed_at":"2026-10-19T16:14:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T16:24:30.764603918Z","timing_metadata_checked_at":"2026-10-19T16:24:30.764603918Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T16:24:30.767451509Z","region_metadata_checked_at":"2026-10-19T16:24:30.767451509Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"26945564d940a2c48d5ae6bbcf8d9d5550355b4b6edcdcc85ad6135e49a5f052","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112177636460466176","occurred_at":"2026-10-19T13:42:45Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}},{"row_id":"ad2226f328423b8aace4470012463b1a0a6cf074f9f18cc5dc2ece3315856390","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112177636460466176","occurred_at":"2026-10-19T13:42:45Z","failed_at":"alert","signature_id":"2e81a638699deccff676fadc9bea465d89c45dd7ef0f11af7ee538a64f2f6d78","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 37 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-vft2k\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-vft2k has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-mxpsb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-mxpsb has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-x658v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-x658v has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-vf6t4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-vf6t4 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-n7j7q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-n7j7q has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-27md8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-27md8 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-284f9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-284f9 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-xk84r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-xk84r has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T15:06:10Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-8s8vm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-8s8vm has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-9zjnm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-9zjnm has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-xkdrg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-xkdrg has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-fdrzp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-fdrzp has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-9gsrl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-9gsrl has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-486j8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-486j8 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-22pvb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-22pvb has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-mm4rn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-mm4rn has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-8nbrp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-8nbrp has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-2fb6m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-2fb6m has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-8bk65\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-8bk65 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-vxx9r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-vxx9r has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-2vsrw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-2vsrw has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-bzjbl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-bzjbl has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-q24x7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-q24x7 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-b5lns\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-b5lns has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-4v9ts\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-4v9ts has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-njzt7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-njzt7 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-n5456\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-n5456 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-49x29\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-49x29 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-5vj7r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-5vj7r has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-6ljqn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-6ljqn has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-7fcjd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-7fcjd has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-47qtx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-47qtx has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-59j4f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-59j4f has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-4gxhr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-4gxhr has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-kpjg2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-kpjg2 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:13Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-cqht6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-cqht6 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-19T15:06:11Z\n Ended: 2026-10-19T15:29:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0466176-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m\", pod=\"kube-controller-manager-85d56cd499-ff847\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thtv10b2qlgq1ue1q40ce21ebfh5jeu-y9e3r7k6y9l6j6m/kube-controller-manager-85d56cd499-ff847 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7606,"pr_state":"closed","pr_sha":"f871711ea58eabfd307be0c3edfb7029093cd0d8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T13:38:55Z","started_at":"2026-10-19T13:38:55Z","completed_at":"2026-10-19T15:44:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T15:51:58.456341998Z","timing_metadata_checked_at":"2026-10-19T15:51:58.456341998Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T15:51:58.45991259Z","region_metadata_checked_at":"2026-10-19T15:51:58.45991259Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"066bf6699a4d3d1baaca16170ecb5ce21e6734ac6e8f112143a2e8b34fa181c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","failed_at":"alert","signature_id":"fa8a6cacdaa7a5856fffdfa8d79789f306612470b3ee02cd5790cbaab8b621f1","test_name":"[aro-hcp-observability] [svc] alert KubeControllerManagerDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-19T14:23:28Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-svc\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:27Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:27Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a2ba93dd0738827129c2396e0399e755beaa4979871f41d5eda27f307b79c65f","failure_pattern":"alert [svc] KubeControllerManagerDown fired","search_query":"alert [svc] KubeControllerManagerDown fired"}},{"row_id":"148b8ce710db2031dd979bd37e1b51b929300442858596365c21280e42495e59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","failed_at":"alert","signature_id":"a017e757469163686646b5fba0b071a928dd5160e44056871a323b9c07705567","test_name":"[aro-hcp-observability] [hcp] alert KubeControllerManagerDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:46:29Z\n Ended: 2026-10-19T15:04:27Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:47:27Z\n Ended: 2026-10-19T15:07:27Z\n Severity: Sev2\n Labels: alertname=\"KubeControllerManagerDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeControllerManager has disappeared from Prometheus target discovery.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b66c7fee67ceccd0ec8a90dea7f05a1d431b6da450c3cf13f7036ae436f2dc62","failure_pattern":"alert [hcp] KubeControllerManagerDown fired","search_query":"alert [hcp] KubeControllerManagerDown fired"}},{"row_id":"6bab16e6594e01f243a9642cdd1952f9ae121cc7c3c6d8752f6f1fe4172b6189","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","failed_at":"alert","signature_id":"5d52b99c1caa3445ce1a5a0b42fb7c49f02aae948a373906d50ae2f022e0f597","test_name":"[aro-hcp-observability] [svc] alert KubeSchedulerDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-19T14:23:34Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-svc\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:35Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:35Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"88b9f437716cb0465d3063d214f7d82310cee5e55f3dc4ff179d878d2ccd19c9","failure_pattern":"alert [svc] KubeSchedulerDown fired","search_query":"alert [svc] KubeSchedulerDown fired"}},{"row_id":"9e161a5ed8d9fb717f85ffd8bd72a425e50c2ef7dc20824af9d7c44e13051395","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","failed_at":"alert","signature_id":"5670409fa53544e23ab06da52caee5a4a9011a439c224ce3c27e12a7e9ddad05","test_name":"[aro-hcp-observability] [svc] alert KubeAPIDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-19T14:23:41Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-svc\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:41Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-19T14:27:41Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d369dd2a49951034c0c53d59118386a695e898c39ded1ccfd379bf7a8c4f5b5a","failure_pattern":"alert [svc] KubeAPIDown fired","search_query":"alert [svc] KubeAPIDown fired"}},{"row_id":"c1f57806bb170ea2ced912f3d97eb0ae6cc0c4ceb2057a8941d690649b8d79ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","failed_at":"alert","signature_id":"efee231c046f70c9a5514c2cb33f5396255a80ececaf9aa1d9008955e15f6822","test_name":"[aro-hcp-observability] [hcp] alert KubeletDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:46:14Z\n Ended: 2026-10-19T15:04:13Z\n Severity: Sev2\n Labels: alertname=\"KubeletDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: Kubelet has disappeared from Prometheus target discovery.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:47:17Z\n Ended: 2026-10-19T15:07:16Z\n Severity: Sev2\n Labels: alertname=\"KubeletDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: Kubelet has disappeared from Prometheus target discovery.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"272f312e5c168a4a2038538015592572b208afed8fa159d39ec9705fefde4353","failure_pattern":"alert [hcp] KubeletDown fired","search_query":"alert [hcp] KubeletDown fired"}},{"row_id":"d7717a11cfdfebcc1f86e1c36eda4ba861c314a087f938f2e37e15e191844c4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","failed_at":"alert","signature_id":"c84bdf2c24901b7b3d2d428c68939f0d21f7d2953227c0a0066c2c9f33f7d7bd","test_name":"[aro-hcp-observability] [hcp] alert KubeSchedulerDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:46:54Z\n Ended: 2026-10-19T15:06:57Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:46:54Z\n Ended: 2026-10-19T15:04:55Z\n Severity: Sev2\n Labels: alertname=\"KubeSchedulerDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeScheduler has disappeared from Prometheus target discovery.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3774249349c1b01f6f6824e299fdbd82fba2f2355f502a02d66d0fca664aeba8","failure_pattern":"alert [hcp] KubeSchedulerDown fired","search_query":"alert [hcp] KubeSchedulerDown fired"}},{"row_id":"e7de0a81321723135f084bb4d43d5002d735b05df2b21e0b795d718a94975145","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","failed_at":"e2e","signature_id":"f980adf8603fc169c4ac17be1c7ca63fdf5ff2e144f6e05dfce3213acbfd3546","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x3eb77839e738\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x3eb77839e738\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}},{"row_id":"f923ac1419e604bc8eb30dc986329a80498a1c05ed597d1d62cffbbbf1114b98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112176673532153856","occurred_at":"2026-10-19T13:38:55Z","failed_at":"alert","signature_id":"0a247216608d08991cd5f8a779520976a1aa61b1f0a06325fa95327fe6ee6a06","test_name":"[aro-hcp-observability] [hcp] alert KubeAPIDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:46:52Z\n Ended: 2026-10-19T15:04:51Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-mgmt-1\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:46:52Z\n Ended: 2026-10-19T15:06:52Z\n Severity: Sev2\n Labels: alertname=\"KubeAPIDown\", cluster=\"ci01-j2153856-mgmt-2\", component=\"kubernetes-infrastructure\", region=\"canadacentral\", severity=\"critical\"\n Description: KubeAPI has disappeared from Prometheus target discovery.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33372092a33bdf2f33019fa8533fe8a97bd128691c334dc9af187a04e2f74a1c","failure_pattern":"alert [hcp] KubeAPIDown fired","search_query":"alert [hcp] KubeAPIDown fired"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["272f312e5c168a4a2038538015592572b208afed8fa159d39ec9705fefde4353","33372092a33bdf2f33019fa8533fe8a97bd128691c334dc9af187a04e2f74a1c","3774249349c1b01f6f6824e299fdbd82fba2f2355f502a02d66d0fca664aeba8","3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","88b9f437716cb0465d3063d214f7d82310cee5e55f3dc4ff179d878d2ccd19c9","a2ba93dd0738827129c2396e0399e755beaa4979871f41d5eda27f307b79c65f","b66c7fee67ceccd0ec8a90dea7f05a1d431b6da450c3cf13f7036ae436f2dc62","d369dd2a49951034c0c53d59118386a695e898c39ded1ccfd379bf7a8c4f5b5a"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112173510980800512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6718,"pr_state":"open","pr_sha":"8fd0b1d03448d4ba4b6caac06ece7f894f33ea8f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T13:26:21Z","started_at":"2026-10-19T13:26:21Z","completed_at":"2026-10-19T15:52:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T16:02:41.206005473Z","timing_metadata_checked_at":"2026-10-19T16:02:41.206005473Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T16:02:41.209795363Z","region_metadata_checked_at":"2026-10-19T16:02:41.209795363Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bec43daa80768679c552c0e4a623897e37cfe6ed0216e54aefafeb557c3634ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112173510980800512","occurred_at":"2026-10-19T13:26:21Z","failed_at":"e2e","signature_id":"120137e1c0d361e1b3322689ca44d5b02756db078d8a85b8e6b75e6d9ffab427","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file\nExpected\n \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9","failure_pattern":"registry.redhat.io credentials not found in pull-secret file","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:197]: registry.redhat.io credentials not found in pull-secret file"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87e80b75be6a897cf9537f14e342dff0c963ffe7363afa40b1ca6f7bd56264f9"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112173242356600832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"6e1223b0c2115a12af3fc9bdbba02642d59c900c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T13:25:17Z","started_at":"2026-10-19T13:25:17Z","completed_at":"2026-10-19T15:33:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T15:41:43.378456185Z","timing_metadata_checked_at":"2026-10-19T15:41:43.378456185Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T15:41:43.393243335Z","region_metadata_checked_at":"2026-10-19T15:41:43.393243335Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c1b9d362f51bbd25aef7c5a81ad671965317eed19988fe146747a04c0e0e9620","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112173242356600832","occurred_at":"2026-10-19T13:25:17Z","failed_at":"e2e","signature_id":"c187448bc769289cb30ffae95a4dd66c338ebeb7dbda51deeb2382c564529a67","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x62872c04378\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x62872c04378\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112172496861007872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7601,"pr_state":"closed","pr_sha":"fd7a4084e07a0c7638492dd265960399c00ce236","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T13:22:20Z","started_at":"2026-10-19T13:22:20Z","completed_at":"2026-10-19T13:42:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T13:45:56.847258968Z","timing_metadata_checked_at":"2026-10-19T13:45:56.847258968Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-19T13:45:56.850537157Z","region_metadata_checked_at":"2026-10-19T14:49:00.407559217Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5a6a7262df2e3e923fd30cdd38bcc550464ee0c666f83b84bf462c3dc729df62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112172496861007872","occurred_at":"2026-10-19T13:22:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112168271166836736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7604,"pr_state":"closed","pr_sha":"35c1994a708e09591727ec4f56c0dbf1f09e0870","final_merged_sha":"35c1994a708e09591727ec4f56c0dbf1f09e0870","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T13:05:30Z","started_at":"2026-10-19T13:05:30Z","completed_at":"2026-10-19T15:30:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T15:41:43.958121634Z","timing_metadata_checked_at":"2026-10-19T15:41:43.958121634Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T15:41:43.960909225Z","region_metadata_checked_at":"2026-10-19T15:41:43.960909225Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5a7fdfb9db40ba9772d1121515abfebe003620be3404bc9b6ae56d752fc6d7ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112168271166836736","occurred_at":"2026-10-19T13:05:30Z","failed_at":"e2e","signature_id":"db8a48ab40d96ea4fb53c96ba6b139542335826edd428ef0ac83fd3e9bc63e79","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a6b532d3880\u003e: \n DNS for route host agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1a6b532d22c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1a6b532d3880\u003e: \n DNS for route host agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1a6b532d22c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dhkfh.apps.aro.v20261001.aza8.j6836736.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"7afec242cc6782f6cfcd3888db68f184169d9307625c0210724936328a0ae750","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7604/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112168271166836736","occurred_at":"2026-10-19T13:05:30Z","failed_at":"e2e","signature_id":"ca2d2cbad258d0e0b68f66bc68f3332bc697e149cae9b0a98284c3671d947cb5","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x25effd432cd8\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x25effd432cd8\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7598,"pr_state":"open","pr_sha":"89dfc9794b598045622b7038334953821d5a4a46","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T12:59:20Z","started_at":"2026-10-19T12:59:20Z","completed_at":"2026-10-19T15:27:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T15:41:43.664918021Z","timing_metadata_checked_at":"2026-10-19T15:41:43.664918021Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T15:41:43.667814311Z","region_metadata_checked_at":"2026-10-19T15:41:43.667814311Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"7bdff128c8f6bb94d8681df383462fbb2937ce3a43a7b30270ca9bb1ce6f341c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","occurred_at":"2026-10-19T12:59:20Z","failed_at":"alert","signature_id":"ced778a05ef7140f9fbc99e57d92ba12b8ad04f552e5d2603281fc04299f4213","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s) (6 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:16:02Z\n Ended: 2026-10-19T14:42:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3999104-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o\", pod=\"cloud-network-config-controller-6bd4b474cb-vrlfj\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o/cloud-network-config-controller-6bd4b474cb-vrlfj has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T14:16:02Z\n Ended: 2026-10-19T14:42:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3999104-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o\", pod=\"cloud-network-config-controller-6bd4b474cb-2kncv\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o/cloud-network-config-controller-6bd4b474cb-2kncv has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T14:16:03Z\n Ended: 2026-10-19T14:42:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3999104-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o\", pod=\"cloud-network-config-controller-6bd4b474cb-fbnwx\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o/cloud-network-config-controller-6bd4b474cb-fbnwx has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T14:16:03Z\n Ended: 2026-10-19T14:42:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3999104-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o\", pod=\"cloud-network-config-controller-6bd4b474cb-bs5m5\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o/cloud-network-config-controller-6bd4b474cb-bs5m5 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T14:16:03Z\n Ended: 2026-10-19T14:42:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3999104-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tht84d4qs8a785c9tlhvqhejsm8ekg7-v6o7e3k6y0b2o1n\", pod=\"cloud-network-config-controller-6db797f7fb-zccvt\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tht84d4qs8a785c9tlhvqhejsm8ekg7-v6o7e3k6y0b2o1n/cloud-network-config-controller-6db797f7fb-zccvt has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T14:16:03Z\n Ended: 2026-10-19T14:42:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3999104-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o\", pod=\"cloud-network-config-controller-6bd4b474cb-dqphz\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tht8gtrach45bq9eke6oba1q2fss001-w5j4t6r0g6q1i4o/cloud-network-config-controller-6bd4b474cb-dqphz has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"87ecedf3fde62401597b1a8a2a782cadf17d9bef8ff0995231f97ee1ac15e0a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","occurred_at":"2026-10-19T12:59:20Z","failed_at":"e2e","signature_id":"7729be770e990dbf343ecf49871e27b862504692b1c6c1526fc07e8d52d20aa5","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x2e8edc927b0\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x2e8edc927b0\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}},{"row_id":"b8b6adb826634717f46c5fc9f1912d469943351d2248190f55db02437afebd65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","occurred_at":"2026-10-19T12:59:20Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T14:15:48Z\n Ended: 2026-10-19T14:29:51Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3999104-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"ffac50959255038ba098fb33a8a7638a61e45e5e4aec4f382163cd93b60e628d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112166731613999104","occurred_at":"2026-10-19T12:59:20Z","failed_at":"e2e","signature_id":"279ce8581e5dd18f302a233fe92088be596d83316a923d7afe6a4828279b16b7","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x106d601dad60\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-dgl2pkrhlxx4\" to finish creating: GET https://rp.j3999104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\",\n \"name\": \"87cddd9b-5ff6-4112-815a-afc1ac87bb65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T14:24:33.557951156Z\",\n \"endTime\": \"2026-10-19T14:43:40.561544029Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine des-encrypt-des-np-rhpbh-rrqrt: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine des-encrypt-des-np-rhpbh-rrqrt: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3999104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\n correlation ID: 504c59a4-77a5-4f21-a6c2-acde02224bf6\n client request ID: fb1b6d77-0e4c-4919-b3ea-36fd2e07f63c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:151]: failed to create nodepool des-np with DES\nUnexpected error:\n \u003c*fmt.wrapError | 0x106d601dad60\u003e: \n failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-dgl2pkrhlxx4\" to finish creating: GET https://rp.j3999104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\",\n \"name\": \"87cddd9b-5ff6-4112-815a-afc1ac87bb65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T14:24:33.557951156Z\",\n \"endTime\": \"2026-10-19T14:43:40.561544029Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine des-encrypt-des-np-rhpbh-rrqrt: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine des-encrypt-des-np-rhpbh-rrqrt: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3999104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/87cddd9b-5ff6-4112-815a-afc1ac87bb65\n correlation ID: 504c59a4-77a5-4f21-a6c2-acde02224bf6\n client request ID: fb1b6d77-0e4c-4919-b3ea-36fd2e07f63c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112163579019923456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7497,"pr_state":"open","pr_sha":"793a8fb63e193e71a604f2de4aca38eabbab8dc9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T12:46:53Z","started_at":"2026-10-19T12:46:53Z","completed_at":"2026-10-19T15:06:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T15:09:31.951740319Z","timing_metadata_checked_at":"2026-10-19T15:09:31.951740319Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T15:09:31.955004009Z","region_metadata_checked_at":"2026-10-19T15:09:31.955004009Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bff3a06e88cbd600b7b85cc92d5d99b58337d6b39a7b3e0446c796a2388f7102","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112163579019923456","occurred_at":"2026-10-19T12:46:53Z","failed_at":"e2e","signature_id":"46e4798aef12db46f28e39faecd964135fd6e1d80259c6a8305854e5da705157","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x2013d3ee0768\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x2013d3ee0768\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7603,"pr_state":"closed","pr_sha":"52be94db858a8677052d853bedbb44aa838cf661","final_merged_sha":"52be94db858a8677052d853bedbb44aa838cf661","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-19T12:28:18Z","started_at":"2026-10-19T12:28:18Z","completed_at":"2026-10-19T14:50:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T14:59:28.052467926Z","timing_metadata_checked_at":"2026-10-19T14:59:28.052467926Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T14:59:28.055850515Z","region_metadata_checked_at":"2026-10-19T14:59:28.055850515Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"0cdda793cac491ea085a1e0b0fa7f3843b67c528b4c07ce48ffa2ea9be6ec7d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","failed_at":"alert","signature_id":"f60e38f83fd59b407778f35f45c625f2f225ee28604ad8f0b54ef8309ee41e82","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T13:49:48Z\n Ended: 2026-10-19T13:59:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7955200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7955200/providers/microsoft.monitor/accounts/hcps-j7955200\", namespace=\"ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"3eeb43ae-342b-4c43-8ac2-1b770f3075a4\"\n Description: Pod ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-19T13:50:49Z\n Ended: 2026-10-19T13:59:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7955200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7955200/providers/microsoft.monitor/accounts/hcps-j7955200\", namespace=\"ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"3eeb43ae-342b-4c43-8ac2-1b770f3075a4\"\n Description: Pod ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"46d1674be3e5b15352f5ddb4a98fb68d6eb20f59f31c0b7985c32356e0b9e96c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","failed_at":"e2e","signature_id":"77845656a2d51a0f7208a5af61597929bc726757f687ffa28f72cf3f1c408b72","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xacc0f2d2440\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-jlnfb8x8vq5b\" to finish creating: GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\",\n \"name\": \"d0f38151-9c16-4736-8305-cfdc47700fc8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T13:29:55.150212587Z\",\n \"endTime\": \"2026-10-19T13:48:55.883771022Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-pscp5-fgcc4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-pscp5-fgcc4 to exist\\nMachine ephemeral-disk-ephemeral-np-pscp5-w4psz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-pscp5-w4psz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-pscp5-fgcc4: Failed: interfaces failed to create or update. err: failed to get existing resource ephemeral-osdisk-jlnfb8x8vq5b--managed/ephemeral-disk-ephemeral-np-pscp5-fgcc4-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Network/networkInterfaces/ephemeral-disk-ephemeral-np-pscp5-fgcc4-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '10' seconds after additional token... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\n correlation ID: 346dd7ce-e606-492f-925a-aa985115ed46\n client request ID: 9eb4247b-08f9-4cb3-a183-7df951ef2271\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:108]: failed to create nodepool ephemeral-np with ephemeral OS disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xacc0f2d2440\u003e: \n failed to create NodePool ephemeral-np: failed waiting for nodepool=\"ephemeral-np\" for cluster \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-jlnfb8x8vq5b\" to finish creating: GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\",\n \"name\": \"d0f38151-9c16-4736-8305-cfdc47700fc8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T13:29:55.150212587Z\",\n \"endTime\": \"2026-10-19T13:48:55.883771022Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine ephemeral-disk-ephemeral-np-pscp5-fgcc4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-pscp5-fgcc4 to exist\\nMachine ephemeral-disk-ephemeral-np-pscp5-w4psz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Compute/virtualMachines/ephemeral-disk-ephemeral-np-pscp5-w4psz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine ephemeral-disk-ephemeral-np-pscp5-fgcc4: Failed: interfaces failed to create or update. err: failed to get existing resource ephemeral-osdisk-jlnfb8x8vq5b--managed/ephemeral-disk-ephemeral-np-pscp5-fgcc4-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-jlnfb8x8vq5b--managed/providers/Microsoft.Network/networkInterfaces/ephemeral-disk-ephemeral-np-pscp5-fgcc4-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '10' seconds after additional token... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d0f38151-9c16-4736-8305-cfdc47700fc8\n correlation ID: 346dd7ce-e606-492f-925a-aa985115ed46\n client request ID: 9eb4247b-08f9-4cb3-a183-7df951ef2271\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"618da575c5b58fd2ef178dda38409b6151172a0ed224c27153d5f47fde89e12b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional token... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ff6d3d81cafcedd168a370fa5b7cfc7e471dba38f2aeab42b864969d05b0a44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","failed_at":"e2e","signature_id":"5b8322b2740f813049ff2c9cf50d132e24ae2a2035200cb00221f24e6219e8d4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4ksqkw\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x875c0f26cc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-4ksqkw: failed waiting for cluster=\"cp-ystream-upgrade-4-21-4ksqkw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-smptp4v58hz8\" to finish creating: GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed4c5b14-08c2-420a-930b-887ecdc0b2b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed4c5b14-08c2-420a-930b-887ecdc0b2b3\",\n \"name\": \"ed4c5b14-08c2-420a-930b-887ecdc0b2b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T13:20:10.282967129Z\",\n \"endTime\": \"2026-10-19T13:49:15.52369038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed4c5b14-08c2-420a-930b-887ecdc0b2b3\n correlation ID: 885d6f5f-5f9d-4eb3-8a13-c8dddc468bfa\n client request ID: 1533b694-7687-4dd7-bde1-4f961ff6ec29\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4ksqkw\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x875c0f26cc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-4ksqkw: failed waiting for cluster=\"cp-ystream-upgrade-4-21-4ksqkw\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-smptp4v58hz8\" to finish creating: GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed4c5b14-08c2-420a-930b-887ecdc0b2b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed4c5b14-08c2-420a-930b-887ecdc0b2b3\",\n \"name\": \"ed4c5b14-08c2-420a-930b-887ecdc0b2b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T13:20:10.282967129Z\",\n \"endTime\": \"2026-10-19T13:49:15.52369038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7955200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed4c5b14-08c2-420a-930b-887ecdc0b2b3\n correlation ID: 885d6f5f-5f9d-4eb3-8a13-c8dddc468bfa\n client request ID: 1533b694-7687-4dd7-bde1-4f961ff6ec29\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b490f68ea4817016ba367020248c3077ede049bf8818a6e68eaabbce4ed65a42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","failed_at":"e2e","signature_id":"571bfa44b18692af70f32970a9dbbe3eb0277ef6a3941e907e7523d8a253cfb0","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x2d77cef36660\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x2d77cef36660\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}},{"row_id":"ee34490b22c357838dcdc0bbe7b61e91c4941b444e558d925ce858c9efcd5d34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","failed_at":"alert","signature_id":"985772ce6b4a5fa9118e8632b816d88f49438c32da40ccd8c2405969b1eff3af","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T13:40:54Z\n Ended: 2026-10-19T14:06:55Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7955200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7955200/providers/microsoft.monitor/accounts/hcps-j7955200\", namespace=\"ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vsdh5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T13:40:54Z\n Ended: 2026-10-19T14:06:55Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7955200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7955200/providers/microsoft.monitor/accounts/hcps-j7955200\", namespace=\"ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vsdh5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"f6077f4341007deb4cd7d246c76af83290ca9935756cba724bd74baadedd2b33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7603/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112158901787955200","occurred_at":"2026-10-19T12:28:18Z","failed_at":"alert","signature_id":"e079f4902db633dcc96bb8366e4fd56a8e30fed4d0a804d743ad8620e1a6ed12","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T13:54:50Z\n Ended: 2026-10-19T14:07:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7955200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7955200/providers/microsoft.monitor/accounts/hcps-j7955200\", namespace=\"ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vsdh5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T13:55:53Z\n Ended: 2026-10-19T14:07:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7955200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7955200/providers/microsoft.monitor/accounts/hcps-j7955200\", namespace=\"ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vsdh5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thsm2jsmop3m17ptumulcji2sfp5cmr-s4t2f9w2p6b9m4v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","618da575c5b58fd2ef178dda38409b6151172a0ed224c27153d5f47fde89e12b","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112155408402485248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7441,"pr_state":"open","pr_sha":"9fd624fa0ec3431e969f365fb218a1c2e88c3aac","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T12:14:25Z","started_at":"2026-10-19T12:14:25Z","completed_at":"2026-10-19T14:27:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T14:38:45.407422432Z","timing_metadata_checked_at":"2026-10-19T14:38:45.407422432Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T14:38:45.410647721Z","region_metadata_checked_at":"2026-10-19T14:38:45.410647721Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a1246a9df6844ebd2a13536269c15c1602d54e5c72f70932e09c33aa6d3efe4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112155408402485248","occurred_at":"2026-10-19T12:14:25Z","failed_at":"e2e","signature_id":"70c1e97b6ee68a8af51a0d1a084c8bb6d1520a0dc4d7df34c5a4b515db5df67b","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x1162a0339230\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x1162a0339230\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112154781085601792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7597,"pr_state":"closed","pr_sha":"d071a83f32db9a874ed681ae192ed18ce7bc5bf0","final_merged_sha":"322d4798e9f946e8950f4ab188bec64d82a6143a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T12:11:56Z","started_at":"2026-10-19T12:11:56Z","completed_at":"2026-10-19T14:23:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T14:28:08.077703363Z","timing_metadata_checked_at":"2026-10-19T14:28:08.077703363Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T14:28:08.081213052Z","region_metadata_checked_at":"2026-10-19T14:28:08.081213052Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9600400b04f7d9ff93ca71a29c14bd2f78f0a3ee5930a476122ac7cd6bea9b8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112154781085601792","occurred_at":"2026-10-19T12:11:56Z","failed_at":"e2e","signature_id":"ae04773ede4b41e57386dc7ca80e3ac46afe158f711be0190a4f51cb38c2dfb1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x34e7945e0180\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x34e7945e0180\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112144183836807168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7065,"pr_state":"open","pr_sha":"b1c442f24e647a96a09fa2938c9466dbc0f43a55","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T11:29:49Z","started_at":"2026-10-19T11:29:49Z","completed_at":"2026-10-19T13:50:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T13:56:50.928080075Z","timing_metadata_checked_at":"2026-10-19T13:56:50.928080075Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T13:56:50.931370864Z","region_metadata_checked_at":"2026-10-19T13:56:50.931370864Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"73000ad2d0639683cf521802ccedb6936a9144f7ef0fc5413bbff856f0529987","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112144183836807168","occurred_at":"2026-10-19T11:29:49Z","failed_at":"e2e","signature_id":"9b37230de007f4358ce6b69050e67aa0a03a63570db97f0b34fc803d04c59567","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x41704badb18\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:193]: failed to parse pull-secret file\nUnexpected error:\n \u003c*json.SyntaxError | 0x41704badb18\u003e: \n invalid character 'M' looking for beginning of value\n {\n msg: \"invalid character 'M' looking for beginning of value\",\n Offset: 1,\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","failure_pattern":"invalid character 'M' looking for beginning of value","search_query":"occurred"}},{"row_id":"ed3d9b3c8ee835b6724f46bcff6d9cc0916a5bda2fb722d5eda88c5b606f6f17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112144183836807168","occurred_at":"2026-10-19T11:29:49Z","failed_at":"e2e","signature_id":"78304e8ecc8c474151eda4c7847024ad884d03b5e6f91186cdb7ae5231364c20","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal\nUnexpected error:\n \u003c*fmt.wrapError | 0x1dd2fd780100\u003e: \n failed to get graph client: identify caller: get token: ClientSecretCredential: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\n {\n msg: \"failed to get graph client: identify caller: get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*fmt.wrapError | 0x1dd2fd7800e0\u003e{\n msg: \"identify caller: get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*fmt.wrapError | 0x1dd2fd7800c0\u003e{\n msg: \"get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*azidentity.AuthenticationFailedError | 0x1dd2fee1c150\u003e{\n RawResponse: nil,\n credType: \"ClientSecretCredential\",\n message: \"server response error:\\n context deadline exceeded\",\n omitResponse: false,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal\nUnexpected error:\n \u003c*fmt.wrapError | 0x1dd2fd780100\u003e: \n failed to get graph client: identify caller: get token: ClientSecretCredential: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\n {\n msg: \"failed to get graph client: identify caller: get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*fmt.wrapError | 0x1dd2fd7800e0\u003e{\n msg: \"identify caller: get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*fmt.wrapError | 0x1dd2fd7800c0\u003e{\n msg: \"get token: ClientSecretCredential: server response error:\\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\",\n err: \u003c*azidentity.AuthenticationFailedError | 0x1dd2fee1c150\u003e{\n RawResponse: nil,\n credType: \"ClientSecretCredential\",\n message: \"server response error:\\n context deadline exceeded\",\n omitResponse: false,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775","failure_pattern":"context deadline exceeded. To troubleshoot, visit \u003curl\u003e","search_query":"context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["3ac9feed4b144b02860172b1452f7ca8829b522155e730d913cf787926abf4dc","ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112139306637201408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7601,"pr_state":"closed","pr_sha":"fd7a4084e07a0c7638492dd265960399c00ce236","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T11:10:26Z","started_at":"2026-10-19T11:10:26Z","completed_at":"2026-10-19T13:17:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T13:25:11.284141214Z","timing_metadata_checked_at":"2026-10-19T13:25:11.284141214Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T13:25:11.287640402Z","region_metadata_checked_at":"2026-10-19T13:25:11.287640402Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ba13b2e57e3aa8ccf95bc78b7585966ecc6c849871c81dae5a9c0d30c7446b91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112139306637201408","occurred_at":"2026-10-19T11:10:26Z","failed_at":"e2e","signature_id":"5d44285e476fc25f2fca3fbecb4c19176951c7cc91f18773f8fc9aab98a817d1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-2qp7lk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xd92a3e87160\u003e: \n DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xd92a3f627c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-2qp7lk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xd92a3e87160\u003e: \n DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xd92a3f627c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5vnqx.apps.aro.x6a5q8q6c4f1a0y.23a8.j7201408.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112123429770498048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7489,"pr_state":"closed","pr_sha":"a800194beb9b2fac67229770cb4864563f1e4d17","final_merged_sha":"a800194beb9b2fac67229770cb4864563f1e4d17","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T10:07:21Z","started_at":"2026-10-19T10:07:21Z","completed_at":"2026-10-19T12:32:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T12:42:58.269238829Z","timing_metadata_checked_at":"2026-10-19T12:42:58.269238829Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T12:42:58.27235942Z","region_metadata_checked_at":"2026-10-19T12:42:58.27235942Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7582,"pr_state":"closed","pr_sha":"7b4008a615923a1f91f2b45f1ce8b234eff01d63","final_merged_sha":"a1dd5b74b0d0373c6ecafbf826ea37646763f7a7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T09:43:19Z","started_at":"2026-10-19T09:43:19Z","completed_at":"2026-10-19T12:01:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T12:11:19.971381232Z","timing_metadata_checked_at":"2026-10-19T12:11:19.971381232Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T12:11:19.987324076Z","region_metadata_checked_at":"2026-10-19T12:11:19.987324076Z"},"failed_at":["alert","e2e"],"failed_test_count":18,"occurrences":[{"row_id":"0bb17a7d8f64fa7be41c319b376946d1868653adfa0cf16c55600fb07fb0b93b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"alert","signature_id":"1bc65a9f283ab33b10f7701db86d6b6980275fa9553c0469ca59ad241483bdfd","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T11:18:35Z\n Ended: 2026-10-19T11:31:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/capi-provider is not progressing for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T11:18:35Z\n Ended: 2026-10-19T11:31:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/capi-provider is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"429fcab9de847861bc73d647c851f740bbfe30154630193b6c0f1c86d0c6c3c8","failure_pattern":"alert [hcp] KubeDeploymentRolloutStuck fired","search_query":"alert [hcp] KubeDeploymentRolloutStuck fired"}},{"row_id":"1188fda493332eb693b671e36858272213feec908db71474bca89617c74c1287","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"alert","signature_id":"2c715e42991a9a3a8d5fdf35df336623aa8ce1ecf9e8c067bf28792e99af1ca0","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 26 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"set-kubelet-parameters-for-scale\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/set-kubelet-parameters-for-scale has not finished or progressed for at least 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"set-kubelet-parameters-for-scale\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/set-kubelet-parameters-for-scale has not finished or progressed for at least 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-19T10:56:20Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-19T10:56:21Z\n Ended: 2026-10-19T11:08:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-19T11:01:19Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-19T11:01:19Z\n Ended: 2026-10-19T11:07:17Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"1ff3115e575cff1993b2471e0c902d89e9a822bf56af22192fe4d11449cf9b00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"e4c0b02a4cead83c39b9290d8769844d1c05d2d84446e896f711e6903b9a464b","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xd36763f9420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-q6wh6zbcv2x2\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\",\n \"name\": \"42423002-b716-49bf-b44c-2e1fd4733997\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:31.508047813Z\",\n \"endTime\": \"2026-10-19T11:00:34.28001697Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\n correlation ID: 1aed53cb-56a1-406f-8bbf-9de9f16e4e1c\n client request ID: 90f415c0-66a9-4a24-b9de-2db2faf18c9c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xd36763f9420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-q6wh6zbcv2x2\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\",\n \"name\": \"42423002-b716-49bf-b44c-2e1fd4733997\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:31.508047813Z\",\n \"endTime\": \"2026-10-19T11:00:34.28001697Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/42423002-b716-49bf-b44c-2e1fd4733997\n correlation ID: 1aed53cb-56a1-406f-8bbf-9de9f16e4e1c\n client request ID: 90f415c0-66a9-4a24-b9de-2db2faf18c9c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"727b90d738458fae42fd743aac06537d1157ba87ee4430dc05269762a9e0b6d7","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"230fcc13d80d0f9cc76316ea69638f84a3345c2b007803fc88b952694f512a4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"17a4d563dd3fd1d5f96570c753ad16a1d18bf9d1bdf8b5b3ea44580c362c8768","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x1c5cb893a690\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-h758j8sh8qhg\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-h758j8sh8qhg\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x1c5cb889c060\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0x1c5cb893a690\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-h758j8sh8qhg\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-h758j8sh8qhg\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x1c5cb889c060\u003e{\n s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-h758j8sh8qhg\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-h758j8sh8qhg, error: context deadline exceeded"}},{"row_id":"2ec2e3e7ce34c20c209a10b7861186dca81b68f8c5c780b36ced7d2746028be0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"14ccb9df6630ab0c7d18ff56028197d76162b4a16d38963003971697eb3da1eb","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x169a1fa14360\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-pvwnm5h2lwhh\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\",\n \"name\": \"201f68b6-906f-4c7f-9b20-18d6e82fa913\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:39.240419381Z\",\n \"endTime\": \"2026-10-19T11:00:44.480106321Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, cluster-node-tuning-operator, redhat-marketplace-catalog, catalog-operator, cluster-network-operator, certified-operators-catalog, olm-collect-profiles, openshift-controller-manager, packageserver, machine-approver, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, openshift-route-controller-manager, community-operators-catalog, cluster-image-registry-operator, dns-operator, konnectivity-agent, olm-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, hosted-cluster-config-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\n correlation ID: a21d9aa7-da4e-4f79-9da1-353faa5bb8d0\n client request ID: 457b6d19-40ff-4036-a44f-729e35535125\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x169a1fa14360\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-pvwnm5h2lwhh\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\",\n \"name\": \"201f68b6-906f-4c7f-9b20-18d6e82fa913\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:39.240419381Z\",\n \"endTime\": \"2026-10-19T11:00:44.480106321Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, cluster-node-tuning-operator, redhat-marketplace-catalog, catalog-operator, cluster-network-operator, certified-operators-catalog, olm-collect-profiles, openshift-controller-manager, packageserver, machine-approver, cluster-storage-operator, cluster-version-operator, cluster-policy-controller, openshift-route-controller-manager, community-operators-catalog, cluster-image-registry-operator, dns-operator, konnectivity-agent, olm-operator, azure-cloud-controller-manager, csi-snapshot-controller-operator, hosted-cluster-config-operator, ingress-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/201f68b6-906f-4c7f-9b20-18d6e82fa913\n correlation ID: a21d9aa7-da4e-4f79-9da1-353faa5bb8d0\n client request ID: 457b6d19-40ff-4036-a44f-729e35535125\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6922af18a79657334aba15a6dc66a7eed9b7eb1fb78e4dc25e99132916eaecff","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3914bf55f70e20417dca849264ed6e69627517d0126055fc4d9c274f8db61051","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"fc3a92da63d1f72fca2a38dd8f0204f04d58ef8149a8800bb69f9a022034a922","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4jxfrx\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b2f64940d00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-4jxfrx: failed waiting for cluster=\"cp-ystream-upgrade-4-21-4jxfrx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dxmr9r2kr8sj\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\",\n \"name\": \"e769daae-d00d-49ae-a5a9-a8aa6f824f09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:38.502730447Z\",\n \"endTime\": \"2026-10-19T11:00:44.46660972Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 23m51s ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\n correlation ID: 72780af9-37e8-42d0-b405-3ed5aea87756\n client request ID: 04371127-7fbb-474b-a1aa-c21583ae4e63\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-4jxfrx\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b2f64940d00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-4jxfrx: failed waiting for cluster=\"cp-ystream-upgrade-4-21-4jxfrx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dxmr9r2kr8sj\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\",\n \"name\": \"e769daae-d00d-49ae-a5a9-a8aa6f824f09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:38.502730447Z\",\n \"endTime\": \"2026-10-19T11:00:44.46660972Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.43 is Partial (want Completed), started 23m51s ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e769daae-d00d-49ae-a5a9-a8aa6f824f09\n correlation ID: 72780af9-37e8-42d0-b405-3ed5aea87756\n client request ID: 04371127-7fbb-474b-a1aa-c21583ae4e63\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"854689b6465ea9b1f99065d40b48241478ad62957ce512515a9e3318a109b484","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"42fbdd0c5a8aaa7b49771549ba558d6df61d31650e34f069a72a12251dcf9872","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"9d277ac26f53ad4a08cdfd07a5091cd11c8dfc2c77ed5fa68ca2993e5b928331","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-ld4f2qxz9tlh/cluster-candidate-4-23-prg9qx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x103e49a6a6a0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-prg9qx: failed waiting for cluster=\"cluster-candidate-4-23-prg9qx\" in resourcegroup=\"rg-candidate-4-23-ld4f2qxz9tlh\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\",\n \"name\": \"87105fe8-f597-4050-9513-e5fec5cf8900\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:31.32598821Z\",\n \"endTime\": \"2026-10-19T11:00:34.343098485Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, cluster-autoscaler, cluster-image-registry-operator, redhat-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, catalog-operator, kube-storage-version-migrator, hosted-cluster-config-operator, olm-collect-profiles, konnectivity-agent, kube-scheduler, machine-approver, cluster-storage-operator, certified-operators-catalog, cluster-node-tuning-operator, community-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, kube-controller-manager, olm-operator, control-plane-pki-operator, ingress-operator, cluster-version-operator, openshift-apiserver, cluster-policy-controller, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\n correlation ID: fc208c85-0c22-4fd2-b0d8-868115a21156\n client request ID: 2c46da4c-f387-4bd8-8b83-56d49c4e1b76\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-ld4f2qxz9tlh/cluster-candidate-4-23-prg9qx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x103e49a6a6a0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-prg9qx: failed waiting for cluster=\"cluster-candidate-4-23-prg9qx\" in resourcegroup=\"rg-candidate-4-23-ld4f2qxz9tlh\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\",\n \"name\": \"87105fe8-f597-4050-9513-e5fec5cf8900\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:31.32598821Z\",\n \"endTime\": \"2026-10-19T11:00:34.343098485Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, cluster-autoscaler, cluster-image-registry-operator, redhat-operators-catalog, azure-cloud-controller-manager, cluster-network-operator, catalog-operator, kube-storage-version-migrator, hosted-cluster-config-operator, olm-collect-profiles, konnectivity-agent, kube-scheduler, machine-approver, cluster-storage-operator, certified-operators-catalog, cluster-node-tuning-operator, community-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, kube-controller-manager, olm-operator, control-plane-pki-operator, ingress-operator, cluster-version-operator, openshift-apiserver, cluster-policy-controller, dns-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87105fe8-f597-4050-9513-e5fec5cf8900\n correlation ID: fc208c85-0c22-4fd2-b0d8-868115a21156\n client request ID: 2c46da4c-f387-4bd8-8b83-56d49c4e1b76\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"847f58c2e694684da512b86721767988d4eb7fbb0f488bd687e65af5d9a8f912","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4fe5233d828ea1a7558069e2b6565d168d879c85d1a30fdddb5a209f98b7f71f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"7ccb09f363018848f8b97a30c2077568c410258b8f6ab2c8e3c4e277049bd960","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x1eb3701d2840\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ljv8hq9fnscx\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/324376be-383b-4d90-8413-08396178b495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/324376be-383b-4d90-8413-08396178b495\",\n \"name\": \"324376be-383b-4d90-8413-08396178b495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:33:06.977502829Z\",\n \"endTime\": \"2026-10-19T11:02:14.171233214Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/324376be-383b-4d90-8413-08396178b495\n correlation ID: f6af9da5-5eba-49f3-9f86-e6e1124bd7f3\n client request ID: 30b2ba1c-045a-4d02-9c28-804fd3a69b7f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x1eb3701d2840\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ljv8hq9fnscx\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/324376be-383b-4d90-8413-08396178b495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/324376be-383b-4d90-8413-08396178b495\",\n \"name\": \"324376be-383b-4d90-8413-08396178b495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:33:06.977502829Z\",\n \"endTime\": \"2026-10-19T11:02:14.171233214Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/324376be-383b-4d90-8413-08396178b495\n correlation ID: f6af9da5-5eba-49f3-9f86-e6e1124bd7f3\n client request ID: 30b2ba1c-045a-4d02-9c28-804fd3a69b7f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"607da5e0aa0423474d7d0acef236a4d234a03e7a6d11b39638ca500c7279b1e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"alert","signature_id":"9f646690f490830019c22ee82a99cd35c179f01219b83c3accb6f681e2db6fe2","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T10:56:32Z\n Ended: 2026-10-19T11:24:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1bb593ff-9c5b-45f3-a6e4-58b2cc1e759f\"\n Description: Pod ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-19T10:56:32Z\n Ended: 2026-10-19T11:24:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1bb593ff-9c5b-45f3-a6e4-58b2cc1e759f\"\n Description: Pod ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"68814c04824499d2e73930753ec88bba1e2df0c446cd09354244340f6ec4b08e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"alert","signature_id":"50734fc067690db24537483069cc361f5320a6cc0719aacfbd83716d2a36feaa","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T10:57:11Z\n Ended: 2026-10-19T11:07:12Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"prometheus\", node=\"aks-userswft1-30082608-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft1-30082608-vmss000000 is unreachable and some workloads may be rescheduled.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T10:57:11Z\n Ended: 2026-10-19T11:07:12Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"prometheus\", node=\"aks-userswft1-30082608-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft1-30082608-vmss000000 is unreachable and some workloads may be rescheduled.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired"}},{"row_id":"75ebf67890f2f112d29c830086d10d028d50def5579cb378752dab0ff127703f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"alert","signature_id":"ed5cd02b1f495df18e96d006d680e79c415f3cff5710741d4cccbefb8851574d","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T10:56:23Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T10:56:23Z\n Ended: 2026-10-19T11:07:18Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/services-j7965568\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired"}},{"row_id":"767e510cefadf41518073b5bef36b54bd0ee91afc52204d19b657fe26192073a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"b208af379d964ed16647d5f8ea8d0ca59bb7ee9c6c4869a7d77f00e95f411925","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-bkqnpw2l8zf5/cluster-candidate-5-0-ccpvpc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d23d4310dc0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-ccpvpc: failed waiting for cluster=\"cluster-candidate-5-0-ccpvpc\" in resourcegroup=\"rg-candidate-5-0-bkqnpw2l8zf5\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\",\n \"name\": \"44c0dc0d-efb0-45b9-885c-ff622849913c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:41.722981857Z\",\n \"endTime\": \"2026-10-19T11:00:44.465792588Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\n correlation ID: 77467a09-84cb-420c-a3bc-027f97f67194\n client request ID: 2b188110-77bd-4bc4-9d94-27a812b7e09a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-bkqnpw2l8zf5/cluster-candidate-5-0-ccpvpc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x3d23d4310dc0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-ccpvpc: failed waiting for cluster=\"cluster-candidate-5-0-ccpvpc\" in resourcegroup=\"rg-candidate-5-0-bkqnpw2l8zf5\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\",\n \"name\": \"44c0dc0d-efb0-45b9-885c-ff622849913c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:31:41.722981857Z\",\n \"endTime\": \"2026-10-19T11:00:44.465792588Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/44c0dc0d-efb0-45b9-885c-ff622849913c\n correlation ID: 77467a09-84cb-420c-a3bc-027f97f67194\n client request ID: 2b188110-77bd-4bc4-9d94-27a812b7e09a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"727b90d738458fae42fd743aac06537d1157ba87ee4430dc05269762a9e0b6d7","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8f0ef37733a545a7ff5261fd9f234b74ed18c6e08f471ea7ed0b181963944e79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"9cf94771ae7bdae1859fe77a8ea6f9b3c3f1d43f1721da6e61995c2a20f5059c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xbe67dc5ca00\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-gs2cw8rqvtsp\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\",\n \"name\": \"9c4a9080-1915-4d6c-8c88-a011da6e522c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:33:35.601658411Z\",\n \"endTime\": \"2026-10-19T11:02:44.344881038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, csi-snapshot-controller-operator, konnectivity-agent, redhat-operators-catalog, certified-operators-catalog, ingress-operator, olm-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, kube-controller-manager, openshift-controller-manager, cluster-network-operator, redhat-marketplace-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-policy-controller, azure-cloud-controller-manager, packageserver, cluster-version-operator, openshift-apiserver, cluster-autoscaler, machine-approver, capi-provider, catalog-operator, community-operators-catalog, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\n correlation ID: ebcf2a7d-e9a4-4dbb-b982-a31f966e3404\n client request ID: d11643b0-b580-4f4f-a16e-c60b01f474c0\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xbe67dc5ca00\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-gs2cw8rqvtsp\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\",\n \"name\": \"9c4a9080-1915-4d6c-8c88-a011da6e522c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:33:35.601658411Z\",\n \"endTime\": \"2026-10-19T11:02:44.344881038Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, csi-snapshot-controller-operator, konnectivity-agent, redhat-operators-catalog, certified-operators-catalog, ingress-operator, olm-operator, cluster-storage-operator, cluster-node-tuning-operator, dns-operator, kube-controller-manager, openshift-controller-manager, cluster-network-operator, redhat-marketplace-catalog, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-policy-controller, azure-cloud-controller-manager, packageserver, cluster-version-operator, openshift-apiserver, cluster-autoscaler, machine-approver, capi-provider, catalog-operator, community-operators-catalog, control-plane-pki-operator; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, capi-provider deployment has 2 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, kube-controller-manager deployment has 2 unavailable replicas, kube-scheduler deployment has 2 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 3 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9c4a9080-1915-4d6c-8c88-a011da6e522c\n correlation ID: ebcf2a7d-e9a4-4dbb-b982-a31f966e3404\n client request ID: d11643b0-b580-4f4f-a16e-c60b01f474c0\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28485ae47900ca737aa0bfad6d457ebe9972f9b9dbd85391d03a17bd274e9f55","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"90cc57a9cef8c4b1bb74273301751203f5a8689a7406b6f6bfc480a5416074c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"cb2937c580cb5b7a220440cf73792f0de178b090817c5fbd61ec46b3faf7c8f1","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x22a4ab41e440\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-tkvqmscnx8hd\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\",\n \"name\": \"7966e350-c6be-4e91-9c2e-1655783cec25\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:32:11.987681126Z\",\n \"endTime\": \"2026-10-19T11:01:14.305309512Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\n correlation ID: 8f412970-2932-49aa-85c1-e501fdb88128\n client request ID: 037bff49-0293-4f3d-ab7f-9ff8b71e3d4a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x22a4ab41e440\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-tkvqmscnx8hd\" to finish creating: GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\",\n \"name\": \"7966e350-c6be-4e91-9c2e-1655783cec25\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:32:11.987681126Z\",\n \"endTime\": \"2026-10-19T11:01:14.305309512Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-marketplace-catalog, redhat-operators-catalog, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7966e350-c6be-4e91-9c2e-1655783cec25\n correlation ID: 8f412970-2932-49aa-85c1-e501fdb88128\n client request ID: 037bff49-0293-4f3d-ab7f-9ff8b71e3d4a\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e15f9141b2e42a01158494cd605cc8736f505f4288f28a00af49203db3d9fd9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9d6f17ef17d2dad4524099c2356143053515f9200bc8279b62b6e4366c95bc6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"8e175b76a31d7caa36a30a13af8ff7ce54e783d826c89b8e5d1babc7577f83a0","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x22128cbeb10\u003e: \n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\",\n \"name\": \"26184216-4ab9-48f4-b1f2-ce27f41ac9be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:32:37.259328289Z\",\n \"endTime\": \"2026-10-19T11:01:44.306693815Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-node-tuning-operator, certified-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, dns-operator, ingress-operator, redhat-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\n correlation ID: 87b43167-8612-41cf-b2af-01e82520620b\n client request ID: 826cfb23-3364-490c-b9f1-549ca623afb4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0x22128cbeb10\u003e: \n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\",\n \"name\": \"26184216-4ab9-48f4-b1f2-ce27f41ac9be\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T10:32:37.259328289Z\",\n \"endTime\": \"2026-10-19T11:01:44.306693815Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: csi-snapshot-controller-operator, cluster-node-tuning-operator, certified-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, dns-operator, ingress-operator, redhat-operators-catalog, cluster-image-registry-operator; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7965568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26184216-4ab9-48f4-b1f2-ce27f41ac9be\n correlation ID: 87b43167-8612-41cf-b2af-01e82520620b\n client request ID: 826cfb23-3364-490c-b9f1-549ca623afb4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e15f9141b2e42a01158494cd605cc8736f505f4288f28a00af49203db3d9fd9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9e276a1cc2193efff633fc75a193dc0b71f81183ceb6eb2a508e2a8c932dcac3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"alert","signature_id":"5f97d4e148771e1613ddbaeca969a031101167ea329462e75b57cd1d7557c9ee","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 66 time(s) (42 unknown, 24 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T10:44:35Z\n Ended: 2026-10-19T10:51:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7oomta392aplhs9dcvdp28vtt5he-cilium-cluster\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7oomta392aplhs9dcvdp28vtt5he-cilium-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T10:44:35Z\n Ended: 2026-10-19T10:51:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq717si3aet42im2u3mgm1p94lgs83-private-kas\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq717si3aet42im2u3mgm1p94lgs83-private-kas/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T10:44:35Z\n Ended: 2026-10-19T10:51:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq800b8r02hfj17953dpa5qhvqgvb7-agg-cluster\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq800b8r02hfj17953dpa5qhvqgvb7-agg-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"cluster-policy-controller-8889f97bf-twmrh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/cluster-policy-controller-8889f97bf-twmrh has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7404tc5dseflpaqmlgdjmpplinrv-s8a1x8h4i2l2k9y\", pod=\"kube-apiserver-85c8fd75b9-pj5z4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7404tc5dseflpaqmlgdjmpplinrv-s8a1x8h4i2l2k9y/kube-apiserver-85c8fd75b9-pj5z4 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"openshift-route-controller-manager-6f94c89644-rpn2h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/openshift-route-controller-manager-6f94c89644-rpn2h has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"csi-snapshot-controller-operator-565fb6b897-52hk6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/csi-snapshot-controller-operator-565fb6b897-52hk6 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"cluster-network-operator-6c88f69f48-9nb2d\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/cluster-network-operator-6c88f69f48-9nb2d has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"cluster-storage-operator-7775d5d868-bp7g4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/cluster-storage-operator-7775d5d868-bp7g4 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq738p1bcdqbp8b2i3gbn005aaui84-basic-cluster\", pod=\"kube-scheduler-86dcc8bf4f-qpbrh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq738p1bcdqbp8b2i3gbn005aaui84-basic-cluster/kube-scheduler-86dcc8bf4f-qpbrh has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u\", pod=\"kube-apiserver-7f8bf74cb4-tvz2n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u/kube-apiserver-7f8bf74cb4-tvz2n has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"control-plane-pki-operator-7ff86fdd89-d56mn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/control-plane-pki-operator-7ff86fdd89-d56mn has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"cluster-image-registry-operator-6646df896c-cj5wk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/cluster-image-registry-operator-6646df896c-cj5wk has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"ingress-operator-65df885c85-5plmd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/ingress-operator-65df885c85-5plmd has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"openshift-controller-manager-5b799fbc74-wklbs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/openshift-controller-manager-5b799fbc74-wklbs has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"cluster-version-operator-665989cf9-tgztw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/cluster-version-operator-665989cf9-tgztw has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"dns-operator-778b6fd76c-ljzf6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/dns-operator-778b6fd76c-ljzf6 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"cluster-node-tuning-operator-55b75b94dc-6tcrk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/cluster-node-tuning-operator-55b75b94dc-6tcrk has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"cluster-network-operator-564c8cd855-j6ww2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/cluster-network-operator-564c8cd855-j6ww2 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"cluster-policy-controller-5c86cfdd85-nn7lg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/cluster-policy-controller-5c86cfdd85-nn7lg has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"cluster-storage-operator-56885bd74c-xwqvp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/cluster-storage-operator-56885bd74c-xwqvp has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq738p1bcdqbp8b2i3gbn005aaui84-basic-cluster\", pod=\"openshift-apiserver-658c9854f7-tszzr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq738p1bcdqbp8b2i3gbn005aaui84-basic-cluster/openshift-apiserver-658c9854f7-tszzr has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"kube-controller-manager-7667c986c-hdvfl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/kube-controller-manager-7667c986c-hdvfl has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-19T10:45:34Z\n Ended: 2026-10-19T10:51:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"cluster-version-operator-6569cb855c-kzf2p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/cluster-version-operator-6569cb855c-kzf2p has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-19T10:51:32Z\n Ended: 2026-10-19T11:23:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq717si3aet42im2u3mgm1p94lgs83-private-kas\", pod=\"router-c5bd85547-nwn6l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq717si3aet42im2u3mgm1p94lgs83-private-kas/router-c5bd85547-nwn6l has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-19T10:51:32Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"openshift-apiserver-86cb9c89c-jhkcd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/openshift-apiserver-86cb9c89c-jhkcd has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-19T10:51:32Z\n Ended: 2026-10-19T11:08:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u\", pod=\"kube-apiserver-7f8bf74cb4-fccnx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u/kube-apiserver-7f8bf74cb4-fccnx has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-19T10:51:32Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq800b8r02hfj17953dpa5qhvqgvb7-agg-cluster\", pod=\"router-5f55f7b569-tslff\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq800b8r02hfj17953dpa5qhvqgvb7-agg-cluster/router-5f55f7b569-tslff has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-19T10:51:32Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7oomta392aplhs9dcvdp28vtt5he-cilium-cluster\", pod=\"router-6959d5b854-9hrjr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7oomta392aplhs9dcvdp28vtt5he-cilium-cluster/router-6959d5b854-9hrjr has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-19T10:51:32Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"openshift-apiserver-7ffff95896-fz5vn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/openshift-apiserver-7ffff95896-fz5vn has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-19T10:51:32Z\n Ended: 2026-10-19T11:19:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"router-6c69dcc6c6-wth6h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/router-6c69dcc6c6-wth6h has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p\", pod=\"kube-apiserver-7689969d44-txkvr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7h85q1ncqh2sbc0rd81ub3s5shc9-g0j4a0v8o8g8t8p/kube-apiserver-7689969d44-txkvr has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq72ld0ojj1mp3uk1lgmrffmlrl99b-l3v8s2l3c9m7o0r\", pod=\"network-node-identity-6586679c4c-pcpd9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq72ld0ojj1mp3uk1lgmrffmlrl99b-l3v8s2l3c9m7o0r/network-node-identity-6586679c4c-pcpd9 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq738p1bcdqbp8b2i3gbn005aaui84-basic-cluster\", pod=\"openshift-apiserver-658c9854f7-vgvdp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq738p1bcdqbp8b2i3gbn005aaui84-basic-cluster/openshift-apiserver-658c9854f7-vgvdp has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq72ld0ojj1mp3uk1lgmrffmlrl99b-l3v8s2l3c9m7o0r\", pod=\"openshift-apiserver-858b75549d-2j2rr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq72ld0ojj1mp3uk1lgmrffmlrl99b-l3v8s2l3c9m7o0r/openshift-apiserver-858b75549d-2j2rr has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:07:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics\", pod=\"kube-apiserver-56967ffcb8-v4xbp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7b23klftd67ff71s751eboi9kk1p-ksm-hcp-metrics/kube-apiserver-56967ffcb8-v4xbp has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:08:39Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq72tljm8flvc8illdor05q2d3jsvu-q1z3g1k8p2m8q3s\", pod=\"kube-apiserver-5fbf5fd755-w7kwc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq72tljm8flvc8illdor05q2d3jsvu-q1z3g1k8p2m8q3s/kube-apiserver-5fbf5fd755-w7kwc has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:18:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u\", pod=\"router-59cf5997f6-nzxvq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u/router-59cf5997f6-nzxvq has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq72tljm8flvc8illdor05q2d3jsvu-q1z3g1k8p2m8q3s\", pod=\"openshift-apiserver-7569f5c7df-dzhsx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq72tljm8flvc8illdor05q2d3jsvu-q1z3g1k8p2m8q3s/openshift-apiserver-7569f5c7df-dzhsx has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:08:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq7404tc5dseflpaqmlgdjmpplinrv-s8a1x8h4i2l2k9y\", pod=\"kube-apiserver-85c8fd75b9-x8l99\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq7404tc5dseflpaqmlgdjmpplinrv-s8a1x8h4i2l2k9y/kube-apiserver-85c8fd75b9-x8l99 has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:07:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq738p1bcdqbp8b2i3gbn005aaui84-basic-cluster\", pod=\"kube-apiserver-985c6d9f9-cs7rf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq738p1bcdqbp8b2i3gbn005aaui84-basic-cluster/kube-apiserver-985c6d9f9-cs7rf has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-19T10:51:33Z\n Ended: 2026-10-19T11:08:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thq72ld0ojj1mp3uk1lgmrffmlrl99b-l3v8s2l3c9m7o0r\", pod=\"kube-apiserver-6ddbf8f745-8tgg9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2thq72ld0ojj1mp3uk1lgmrffmlrl99b-l3v8s2l3c9m7o0r/kube-apiserver-6ddbf8f745-8tgg9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"a578afa1241442db577f214bcd8ba0041f8b215a7c5a3e759ebcaae23678d8a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"alert","signature_id":"5823d7990345a57a295b5c723408757e3fd02ee10ebd01906262ea2620d0d364","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T11:08:39Z\n Ended: 2026-10-19T11:31:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T11:08:39Z\n Ended: 2026-10-19T11:31:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-19T11:11:35Z\n Ended: 2026-10-19T11:19:39Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-19T11:11:35Z\n Ended: 2026-10-19T11:18:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-19T11:11:35Z\n Ended: 2026-10-19T11:19:39Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq717si3aet42im2u3mgm1p94lgs83-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thq717si3aet42im2u3mgm1p94lgs83-private-kas/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-19T11:11:35Z\n Ended: 2026-10-19T11:23:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq717si3aet42im2u3mgm1p94lgs83-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thq717si3aet42im2u3mgm1p94lgs83-private-kas/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-19T11:11:35Z\n Ended: 2026-10-19T11:18:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thq70r9o3n24hq38t4ueapic16a3d74-a3v0z7g8c2d5e0u/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-19T11:11:35Z\n Ended: 2026-10-19T11:19:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7965568-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.25:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7965568/providers/microsoft.monitor/accounts/hcps-j7965568\", namespace=\"ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mmhvv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2thq73tilorccuv8p9876u7h0dp8pm57-j9m4x7q3m8f8l0e/router has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ba26ab64c4f0debe8dc849a4e0946a14f5ca61d1243bb8e55129300da07c715f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112117383177965568","occurred_at":"2026-10-19T09:43:19Z","failed_at":"e2e","signature_id":"7573d0a5201f4ebb34194e6054fbfa690d8fbb2ada0cef80ed7ad8a5e9897197","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:136]: failed to verify HCP cluster \"cluster-candidate-4-21-d6zdrd\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0xde74c1ac0a8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\n {\n errs: [\n \u003c*fmt.wrapError | 0xde74c2c7be0\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xde74c2c7bc0\u003e{\n msg: \"failed to list services: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xde74be74ff0\u003e{\n Op: \"Get\",\n URL: \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n Err: \u003c*net.OpError | 0xde74bbe7a90\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xde74b874e10\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xde74b4feb80\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xde74b4feb40\u003e{\n msg: \"failed to list all APIServices: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xde74be1b410\u003e{\n Op: \"Get\",\n URL: \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n Err: \u003c*net.OpError | 0xde74b874fa0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xde74b874e10\u003e0xde74b874e10 (cyclic reference),\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:136]: failed to verify HCP cluster \"cluster-candidate-4-21-d6zdrd\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0xde74c1ac0a8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\n {\n errs: [\n \u003c*fmt.wrapError | 0xde74c2c7be0\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xde74c2c7bc0\u003e{\n msg: \"failed to list services: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xde74be74ff0\u003e{\n Op: \"Get\",\n URL: \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n Err: \u003c*net.OpError | 0xde74bbe7a90\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xde74b874e10\u003e{\n UnwrapErr: nil,\n Err: \"read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n Name: \"api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud\",\n Server: \"172.30.0.10:53\",\n IsTimeout: true,\n IsTemporary: true,\n IsNotFound: false,\n },\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0xde74b4feb80\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*fmt.wrapError | 0xde74b4feb40\u003e{\n msg: \"failed to list all APIServices: Get \\\"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout\",\n err: \u003c*url.Error | 0xde74be1b410\u003e{\n Op: \"Get\",\n URL: \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n Err: \u003c*net.OpError | 0xde74b874fa0\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0xde74b874e10\u003e0xde74b874e10 (cyclic reference),\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"281d6f2e10296c7900f78767c52e72d480329fd4f2572568eaae28e31a3fe696","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t5f2q1v9y1k8g4h.3m43.j7965568.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.96.92:35750-\u003e172.30.0.10:53: i/o timeout"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","1e15f9141b2e42a01158494cd605cc8736f505f4288f28a00af49203db3d9fd9","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","281d6f2e10296c7900f78767c52e72d480329fd4f2572568eaae28e31a3fe696","28485ae47900ca737aa0bfad6d457ebe9972f9b9dbd85391d03a17bd274e9f55","429fcab9de847861bc73d647c851f740bbfe30154630193b6c0f1c86d0c6c3c8","519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","6922af18a79657334aba15a6dc66a7eed9b7eb1fb78e4dc25e99132916eaecff","727b90d738458fae42fd743aac06537d1157ba87ee4430dc05269762a9e0b6d7","847f58c2e694684da512b86721767988d4eb7fbb0f488bd687e65af5d9a8f912","854689b6465ea9b1f99065d40b48241478ad62957ce512515a9e3318a109b484","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112099918813335552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7592,"pr_state":"closed","pr_sha":"fe368ee523373c854ea4f569cb7764366d8e1ea5","final_merged_sha":"c9a469c0a42acf6482344c2525dfc92f27189b30","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-19T08:33:56Z","started_at":"2026-10-19T08:33:56Z","completed_at":"2026-10-19T10:59:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T11:09:47.170261294Z","timing_metadata_checked_at":"2026-10-19T11:09:47.170261294Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T11:09:47.173677183Z","region_metadata_checked_at":"2026-10-19T11:09:47.173677183Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112094774411399168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7599,"pr_state":"closed","pr_sha":"f36dec15838ec28b266a6ab120d1015ca19aad2a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T08:13:10Z","started_at":"2026-10-19T08:13:10Z","completed_at":"2026-10-19T10:26:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T10:38:35.996926855Z","timing_metadata_checked_at":"2026-10-19T10:38:35.996926855Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T10:38:36.005247427Z","region_metadata_checked_at":"2026-10-19T10:38:36.005247427Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a651ff3514c65dd56107edf7344df99cdb72da79b5b142acfe84c0721d2c3e35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112094774411399168","occurred_at":"2026-10-19T08:13:10Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T09:12:06Z\n Ended: 2026-10-19T09:30:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1399168-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112085059526725632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"open","pr_sha":"8882c590ef3846b2c654442660a152aec1a8a512","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T07:34:53Z","started_at":"2026-10-19T07:34:53Z","completed_at":"2026-10-19T10:17:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T10:28:28.702504808Z","timing_metadata_checked_at":"2026-10-19T10:28:28.702504808Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T10:28:28.712523778Z","region_metadata_checked_at":"2026-10-19T10:28:28.712523778Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"80de38a3504b81b72ebefae88c6686704f6ec20ff61661325358cac2fc63f450","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112085059526725632","occurred_at":"2026-10-19T07:34:53Z","failed_at":"e2e","signature_id":"16703b61dd856f6d7b5d00dca02cd82935f47a24ba99f70f8786b2796812a181","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x59ce69d41c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x59ce69eea20\u003e{\n s: \"dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x59ce69d41c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-tjz8vhdkt57n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x59ce69eea20\u003e{\n s: \"dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-2thol4rfhgvi5rmvh2p2p3orcjrvp1s4-g7g2x5z2p7o8m8e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d74615ba0958ced07d8ca078f31c82f56392409cb82a9380566bf2ff546c6cd4","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: dev/centralus/ci01-j6725632-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d74615ba0958ced07d8ca078f31c82f56392409cb82a9380566bf2ff546c6cd4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112079912079724544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7065,"pr_state":"open","pr_sha":"f79863d00c67b17da15a83c33bcd6585ecd0abbe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-19T07:14:26Z","started_at":"2026-10-19T07:14:26Z","completed_at":"2026-10-19T09:25:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T09:36:13.461822438Z","timing_metadata_checked_at":"2026-10-19T09:36:13.461822438Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T09:36:13.465446326Z","region_metadata_checked_at":"2026-10-19T09:36:13.465446326Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7597,"pr_state":"closed","pr_sha":"d071a83f32db9a874ed681ae192ed18ce7bc5bf0","final_merged_sha":"322d4798e9f946e8950f4ab188bec64d82a6143a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T07:05:12Z","started_at":"2026-10-19T07:05:12Z","completed_at":"2026-10-19T09:27:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T09:36:13.599403885Z","timing_metadata_checked_at":"2026-10-19T09:36:13.599403885Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T09:36:13.602625875Z","region_metadata_checked_at":"2026-10-19T09:36:13.602625875Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0aa8ec783832f3fc851fb6a7aaa5b72cf2293595df50cbeee3f917da2db2910b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","occurred_at":"2026-10-19T07:05:12Z","failed_at":"alert","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-19T08:12:56Z\n Ended: 2026-10-19T08:32:56Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1968640-svc\", component=\"backend\", name=\"csstatedump\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-19T08:12:56Z\n Ended: 2026-10-19T08:31:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1968640-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"2acd85dea2bdd188687041cc905c0b56c8e3b8481a3a14f6354bd1baf793d821","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","occurred_at":"2026-10-19T07:05:12Z","failed_at":"e2e","signature_id":"95ce875d62f45d254fefee15188f37e24bc247d864e7512f855578248bd38bb4","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:373]: expected 2 ready and schedulable nodes after the cordon/drain/uncordon cycle\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3f000f2687f0\u003e: \n expected 2 ready (and schedulable) nodes in node pool \"cordon-np\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]\n {\n s: \"expected 2 ready (and schedulable) nodes in node pool \\\"cordon-np\\\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:373]: expected 2 ready and schedulable nodes after the cordon/drain/uncordon cycle\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3f000f2687f0\u003e: \n expected 2 ready (and schedulable) nodes in node pool \"cordon-np\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]\n {\n s: \"expected 2 ready (and schedulable) nodes in node pool \\\"cordon-np\\\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"25e28b5e77d38ab357912804c3c34e80b2c640b1e778e31291607291be7016d6","failure_pattern":"expected \u003ccount\u003e ready (and schedulable) nodes, found \u003ccount\u003e; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [\u003cnode\u003e, \u003cnode\u003e]","search_query":"expected 2 ready (and schedulable) nodes in node pool \"cordon-np\", found 0; matching nodes: ; all nodes by pool (2 total): \u003cno-nodepool-label\u003e: [f8d4b9b3r0n2i8f-cordon-np-h6xll-c5xxx, f8d4b9b3r0n2i8f-cordon-np-h6xll-k9728]"}},{"row_id":"7d889710b2fd5ae6124b9319b73fab7ef88e016c06a7e0c7c1d6cf80137086d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","occurred_at":"2026-10-19T07:05:12Z","failed_at":"e2e","signature_id":"f9ddbe60cb32d9a3ac38695dbd1398ef4cb5516959103f1774c946569ab2ace1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e0c6d55a40\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1e0c6d55a20\u003e{\n msg: \"failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1e0c6d55a00\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e0c6d55a40\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1e0c6d55a20\u003e{\n msg: \"failed to get DaemonSet kube-system/global-pull-secret-syncer: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1e0c6d55a00\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"c0f22fd3e5e474002d4b93a17ecbe89a500e0719e728750fb3a9ef6cb5bcad02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7597/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112077587801968640","occurred_at":"2026-10-19T07:05:12Z","failed_at":"e2e","signature_id":"ae7f359019eb8ec65c1a72a87e13d829e08e16887d3280b98246e0e23d7fa164","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.005s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: 0\nto be \u003e=\n \u003cint\u003e: 2\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.005s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: 0\nto be \u003e=\n \u003cint\u003e: 2","failure_pattern_match":{"status":"clustered","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","25e28b5e77d38ab357912804c3c34e80b2c640b1e778e31291607291be7016d6","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112075402955460608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"closed","pr_sha":"001d1f783bc748e912b648732f913c8049f858d0","final_merged_sha":"b69b4de64af83e2b15f18c7fbc58503c89002461","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T06:56:31Z","started_at":"2026-10-19T06:56:31Z","completed_at":"2026-10-19T07:39:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T07:50:59.478327617Z","timing_metadata_checked_at":"2026-10-19T07:50:59.478327617Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T07:50:59.481964206Z","region_metadata_checked_at":"2026-10-19T07:50:59.481964206Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"12356e34de75baa7c4ca9ced309fc65279d77d1da8956dbb5c9857317fd097d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112075402955460608","occurred_at":"2026-10-19T06:56:31Z","failed_at":"provision","signature_id":"bda1134e26c870bac63ca65499e242da0400c142162bed259a46288d26e4a236","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/storageclass (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-19T07:33:05.070Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 description=\"Step storageclass\\n Kind: Helm\\n\"\ntime=2026-10-19T07:33:05.599Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 values=\"map[skuName:PremiumV2_LRS unannotateJobImage:mcr.microsoft.com/aks/command/runtime@sha256:5c16384cf45be47415214887ce360da268c4e7dbd157914665ad1244e8982595]\"\ntime=2026-10-19T07:33:05.599Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T07:33:05.710Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 namespace=default-sc\ntime=2026-10-19T07:33:05.849Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 dryRun=false\ntime=2026-10-19T07:33:05.849Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T07:33:05.925Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1\ntime=2026-10-19T07:33:56.641Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-19T07:33:56.642Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=storageclass stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/default-sc/unannotate-default-sc not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cff3c23db22aaa97e0ccea58aa1838b5a34f66ed50a6ffa4825b14757281f67"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7596/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112066024999227392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7596,"pr_state":"closed","pr_sha":"a8a4d0bab7e0b460e96aa412c514dc8874b89574","final_merged_sha":"a8a4d0bab7e0b460e96aa412c514dc8874b89574","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T06:19:15Z","started_at":"2026-10-19T06:19:15Z","completed_at":"2026-10-19T08:35:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T08:43:57.526472289Z","timing_metadata_checked_at":"2026-10-19T08:43:57.526472289Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T08:43:57.529815178Z","region_metadata_checked_at":"2026-10-19T08:43:57.529815178Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112019392656052224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7504,"pr_state":"open","pr_sha":"ff90e86c4164b3e9fea89213d701c3f0e7a27058","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-19T03:13:57Z","started_at":"2026-10-19T03:13:57Z","completed_at":"2026-10-19T05:27:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T05:44:58.271444049Z","timing_metadata_checked_at":"2026-10-19T05:44:58.271444049Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T05:44:58.275272936Z","region_metadata_checked_at":"2026-10-19T05:44:58.275272936Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1eae3f1b2ca5d29e561e9a2f24f131a89ec4c6e4459f1ca7efbfd70866cb3ce2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112019392656052224","occurred_at":"2026-10-19T03:13:57Z","failed_at":"e2e","signature_id":"eb8dcd63f6fa8106b7aefdf7658de4b7ab552e999feda4f0ea87bdb655d6430c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x54e9aa24000\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-4wl22sb8l6px\" to finish creating: GET https://rp.j6052224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47d30f4f-4041-41a8-ac8b-37f3547ef206\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47d30f4f-4041-41a8-ac8b-37f3547ef206\",\n \"name\": \"47d30f4f-4041-41a8-ac8b-37f3547ef206\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T04:11:42.972869191Z\",\n \"endTime\": \"2026-10-19T04:30:45.485858922Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-d4j2g-r4b85: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-4wl22sb8l6px--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-d4j2g-r4b85 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-d4j2g-r4b85: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-4wl22sb8l6px--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-d4j2g-r4b85 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6052224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47d30f4f-4041-41a8-ac8b-37f3547ef206\n correlation ID: 2b14871c-7eb2-49b8-a678-9dcdb4e96cad\n client request ID: aa1ac31a-55fd-4750-9293-aaaca548fae3\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x54e9aa24000\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-4wl22sb8l6px\" to finish creating: GET https://rp.j6052224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47d30f4f-4041-41a8-ac8b-37f3547ef206\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47d30f4f-4041-41a8-ac8b-37f3547ef206\",\n \"name\": \"47d30f4f-4041-41a8-ac8b-37f3547ef206\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-19T04:11:42.972869191Z\",\n \"endTime\": \"2026-10-19T04:30:45.485858922Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-d4j2g-r4b85: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-4wl22sb8l6px--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-d4j2g-r4b85 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-d4j2g-r4b85: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-4wl22sb8l6px--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-d4j2g-r4b85 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6052224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/47d30f4f-4041-41a8-ac8b-37f3547ef206\n correlation ID: 2b14871c-7eb2-49b8-a678-9dcdb4e96cad\n client request ID: aa1ac31a-55fd-4750-9293-aaaca548fae3","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-9bqdtwnxktt9\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7595/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112006820758294528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7595,"pr_state":"closed","pr_sha":"a9222e4142adea1bb7dc67e9fdad5e687f567177","final_merged_sha":"a9222e4142adea1bb7dc67e9fdad5e687f567177","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T02:23:59Z","started_at":"2026-10-19T02:23:59Z","completed_at":"2026-10-19T04:35:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T04:54:04.137524855Z","timing_metadata_checked_at":"2026-10-19T04:54:04.137524855Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T04:54:04.141042443Z","region_metadata_checked_at":"2026-10-19T04:54:04.141042443Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111995993326096384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7580,"pr_state":"closed","pr_sha":"cc68e5f22f90cd600ec1702fd6051be82c4f0a33","final_merged_sha":"cc68e5f22f90cd600ec1702fd6051be82c4f0a33","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-19T01:40:58Z","started_at":"2026-10-19T01:40:58Z","completed_at":"2026-10-19T03:58:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-19T04:02:03.350376311Z","timing_metadata_checked_at":"2026-10-19T04:02:03.350376311Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-19T04:02:03.353341101Z","region_metadata_checked_at":"2026-10-19T04:02:03.353341101Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}