{"meta":{"date":"2026-10-24","timezone":"UTC","generated_at":"2026-10-30T05:07:26Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":42,"failed_runs":29,"runs_with_occurrences":29,"runs_with_failure_pattern_matches":22,"runs_with_unmatched_occurrences":7,"runs_without_occurrence_rows":13,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7748/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114134939422167040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7748,"pr_state":"closed","pr_sha":"96712e19d192998d1be01247132b7b43665be4bc","final_merged_sha":"96712e19d192998d1be01247132b7b43665be4bc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T23:20:22Z","started_at":"2026-10-24T23:20:22Z","completed_at":"2026-10-25T01:48:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-25T02:02:22.140130352Z","timing_metadata_checked_at":"2026-10-25T02:02:22.140130352Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-25T02:02:22.143824041Z","region_metadata_checked_at":"2026-10-25T02:02:22.143824041Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114076194226835456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7746,"pr_state":"closed","pr_sha":"887b8bc70a7070905a60851e8342fddd7e2e5578","final_merged_sha":"887b8bc70a7070905a60851e8342fddd7e2e5578","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T19:26:56Z","started_at":"2026-10-24T19:26:56Z","completed_at":"2026-10-24T21:28:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T21:41:04.580940118Z","timing_metadata_checked_at":"2026-10-24T21:41:04.580940118Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T21:41:04.583897809Z","region_metadata_checked_at":"2026-10-24T21:41:04.583897809Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114076081622355968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7742,"pr_state":"closed","pr_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","final_merged_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T19:26:30Z","started_at":"2026-10-24T19:26:30Z","completed_at":"2026-10-24T22:09:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T22:22:59.046085816Z","timing_metadata_checked_at":"2026-10-24T22:22:59.046085816Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T22:22:59.053006496Z","region_metadata_checked_at":"2026-10-24T22:22:59.053006496Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7744/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114072576199233536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7744,"pr_state":"closed","pr_sha":"444bf9b9832a66e1544bc9410a7d27389efa4d06","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-24T19:12:34Z","started_at":"2026-10-24T19:12:34Z","completed_at":"2026-10-24T21:31:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T21:41:04.443267744Z","timing_metadata_checked_at":"2026-10-24T21:41:04.443267744Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T21:41:04.446864733Z","region_metadata_checked_at":"2026-10-24T21:41:04.446864733Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114061342737436672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7742,"pr_state":"closed","pr_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","final_merged_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T18:27:56Z","started_at":"2026-10-24T18:27:56Z","completed_at":"2026-10-24T18:28:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T18:44:06.438585517Z","timing_metadata_checked_at":"2026-10-24T18:44:06.438585517Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-24T18:44:06.443507403Z","region_metadata_checked_at":"2026-10-24T19:47:00.198654052Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6f9efe65badf85ab6f2cafc657a26597d155de3a75630f3b69ecade37a1ea7cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114061342737436672","occurred_at":"2026-10-24T18:27:56Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114058315167174656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7736,"pr_state":"closed","pr_sha":"ed7d121617ab665039646cdf53c36414616c413e","final_merged_sha":"ed7d121617ab665039646cdf53c36414616c413e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T18:15:54Z","started_at":"2026-10-24T18:15:54Z","completed_at":"2026-10-24T20:06:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T20:17:55.087924848Z","timing_metadata_checked_at":"2026-10-24T20:17:55.087924848Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T20:17:55.093424631Z","region_metadata_checked_at":"2026-10-24T20:17:55.093424631Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c3c1cd840a54fe72d4430ac219508591ab20f00199e4e6c4301ab1ea59d48023","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114058315167174656","occurred_at":"2026-10-24T18:15:54Z","failed_at":"e2e","signature_id":"e495e0c4e0c96db7a7c9023b14417bd105afffe14e2c1a410c49514991caf469","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-df9q9l\"\nUnexpected error:\n \u003c*errors.joinError | 0x124e8626e318\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\": nodes \"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x124e8515f040\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\": nodes \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x124e8515f000\u003e{\n msg: \"failed to label node r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\": nodes \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x124e8515efc0\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\": nodes \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x124e8530e280\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-df9q9l\"\nUnexpected error:\n \u003c*errors.joinError | 0x124e8626e318\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\": nodes \"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x124e8515f040\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\": nodes \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x124e8515f000\u003e{\n msg: \"failed to label node r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\": nodes \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x124e8515efc0\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\": nodes \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x124e8530e280\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"r2v6x5y1r4f5h5a-np-df9q9l-fvq4n-7rzfb\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\": nodes \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114056756412813312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7741,"pr_state":"closed","pr_sha":"7c26030ae8e63905ce0f8c90c6f96f0aee339b6b","final_merged_sha":"7c26030ae8e63905ce0f8c90c6f96f0aee339b6b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T18:09:42Z","started_at":"2026-10-24T18:09:42Z","completed_at":"2026-10-24T20:50:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T20:58:52.795581377Z","timing_metadata_checked_at":"2026-10-24T20:58:52.795581377Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T20:58:52.798637968Z","region_metadata_checked_at":"2026-10-24T20:58:52.798637968Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7743/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114056186943770624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7743,"pr_state":"closed","pr_sha":"76393c6b2b338ac96801f17fc2c0c396a096cda2","final_merged_sha":"76393c6b2b338ac96801f17fc2c0c396a096cda2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T18:07:26Z","started_at":"2026-10-24T18:07:26Z","completed_at":"2026-10-24T20:20:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T20:28:07.142443376Z","timing_metadata_checked_at":"2026-10-24T20:28:07.142443376Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T20:28:07.146430362Z","region_metadata_checked_at":"2026-10-24T20:28:07.146430362Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114055934392143872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7742,"pr_state":"closed","pr_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","final_merged_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T18:06:26Z","started_at":"2026-10-24T18:06:26Z","completed_at":"2026-10-24T18:06:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T18:23:02.823130761Z","timing_metadata_checked_at":"2026-10-24T18:23:02.823130761Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-24T18:23:02.826920046Z","region_metadata_checked_at":"2026-10-24T19:25:20.012624868Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a57c7773d09e28017fadfed7805e1737b439e30ee4c830f4b4e2e586a6802757","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114055934392143872","occurred_at":"2026-10-24T18:06:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114055386943197184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7742,"pr_state":"closed","pr_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","final_merged_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T18:04:16Z","started_at":"2026-10-24T18:04:16Z","completed_at":"2026-10-24T18:04:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T18:12:49.619415946Z","timing_metadata_checked_at":"2026-10-24T18:12:49.619415946Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-24T18:12:49.622729234Z","region_metadata_checked_at":"2026-10-24T19:15:13.871098281Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"14728ea8c6fb8af881ab381cc21b98a2971b850fb009f4bcdf696c0b54e8c191","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114055386943197184","occurred_at":"2026-10-24T18:04:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114047539169726464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7730,"pr_state":"open","pr_sha":"e1befdfc016b103d439008ff1990f6c52d510bda","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T17:33:05Z","started_at":"2026-10-24T17:33:05Z","completed_at":"2026-10-24T19:39:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T19:47:00.07921391Z","timing_metadata_checked_at":"2026-10-24T19:47:00.07921391Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T19:47:00.084163895Z","region_metadata_checked_at":"2026-10-24T19:47:00.084163895Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"87d509a7e7416f52586ca39d71637c2d1ba6744a39654f3be23626c95891e1c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114047539169726464","occurred_at":"2026-10-24T17:33:05Z","failed_at":"e2e","signature_id":"685d28f37bdd0fa7fcacd74adb60aa052fa898956388730f22231c7fb6a1c8ba","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x11989c84120\u003e: \n DNS for route host agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x11989c840a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x11989c84120\u003e: \n DNS for route host agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x11989c840a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-kb9js.apps.aro.ingress-cert-le.a11n.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7c740bf3534ca5945b9fa94518bc4b7a520a009eef4fbd7a1fc2602d0f2b958","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-kb9js"}},{"row_id":"cb349cf2de57114234751e0d6f0a6bdd02f756199ed7dfaec6d176731b58398c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114047539169726464","occurred_at":"2026-10-24T17:33:05Z","failed_at":"e2e","signature_id":"0f91636213bd7ade3c925588546db5d56e98c1789eec3db993e11328cd2e8d45","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-zdl7h9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x7f0cb444140\u003e: \n DNS for route host agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x7f0cb818340\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-zdl7h9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x7f0cb444140\u003e: \n DNS for route host agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x7f0cb818340\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lplgk.apps.aro.y6v4u1u9i5a4q3x.e0z9.j9726464.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f7c740bf3534ca5945b9fa94518bc4b7a520a009eef4fbd7a1fc2602d0f2b958"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114041599448059904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7742,"pr_state":"closed","pr_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","final_merged_sha":"f1ae6c51beff69a981b314e77cecdf8426e68a96","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T17:09:28Z","started_at":"2026-10-24T17:09:28Z","completed_at":"2026-10-24T17:23:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T17:30:59.255000522Z","timing_metadata_checked_at":"2026-10-24T17:30:59.255000522Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-24T17:30:59.257942213Z","region_metadata_checked_at":"2026-10-24T18:33:16.999223882Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c94cd38a1b9c1c0e984d79f5f4c09e9961cbb132cc3e7c75352f9359211b62da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114041599448059904","occurred_at":"2026-10-24T17:09:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114027174305468416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7736,"pr_state":"closed","pr_sha":"ed7d121617ab665039646cdf53c36414616c413e","final_merged_sha":"ed7d121617ab665039646cdf53c36414616c413e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T16:12:09Z","started_at":"2026-10-24T16:12:09Z","completed_at":"2026-10-24T17:17:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T17:30:58.950231341Z","timing_metadata_checked_at":"2026-10-24T17:30:58.950231341Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T17:30:58.95383333Z","region_metadata_checked_at":"2026-10-24T17:30:58.95383333Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"bf19bc11edd27148dc1627f70584f3ccc1eedef0e113682fc0d00c6d69156f4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114027174305468416","occurred_at":"2026-10-24T16:12:09Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-24T16:56:57.345Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T16:56:58.695Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5468416-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T16:56:58.695Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5468416-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T16:56:59.317Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5468416-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T16:56:59.317Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j5468416-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T16:56:59.317Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j5468416-svc\ntime=2026-10-24T17:12:01.792Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}},{"row_id":"ffaa8896bf9c0f1f79594decf78fb05f38bce1ca3b5e6fa52d21bc19d193503d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114027174305468416","occurred_at":"2026-10-24T16:12:09Z","failed_at":"alert","signature_id":"0192b4d7195c586e436e00e86981f587ebb031c4ce49ca09161b7c9b266d7d44","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 26 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-pbtkv\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-pbtkv has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-82j8b\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-82j8b has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-f27gs\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-f27gs has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-lwgdc\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-lwgdc has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-vs5np\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-vs5np has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-87dfl\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-87dfl has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-qpl82\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-qpl82 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-5q9wz\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-5q9wz has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-q7t7r\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-q7t7r has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-7llr5\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-7llr5 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-lxj96\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-lxj96 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-mcdmv\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-mcdmv has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-swlsf\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-swlsf has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-tkgxq\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-tkgxq has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-92mtz\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-92mtz has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-7vvmn\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-7vvmn has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-tb8wn\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-tb8wn has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-cv95p\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-cv95p has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-qpvlz\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-qpvlz has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-zfl7b\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-zfl7b has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-4wdk9\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-4wdk9 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-9hkcm\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-9hkcm has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-699hx\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-699hx has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-hdfbt\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-hdfbt has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Fired (not resolved)\n Started: 2026-10-24T17:02:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-cxrkz\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-cxrkz has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Fired (not resolved)\n Started: 2026-10-24T17:03:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5468416-svc\", component=\"kubernetes-infrastructure\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-85fb9c778c-4hnj4\", region=\"centralus\", severity=\"warning\"\n Description: Pod aro-hcp-exporter/aro-hcp-exporter-85fb9c778c-4hnj4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114009717494779904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7741,"pr_state":"closed","pr_sha":"95b7dc25c4967961f7e7ea056397af1375189f54","final_merged_sha":"7c26030ae8e63905ce0f8c90c6f96f0aee339b6b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-24T15:02:47Z","started_at":"2026-10-24T15:02:47Z","completed_at":"2026-10-24T17:08:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T17:20:50.696007465Z","timing_metadata_checked_at":"2026-10-24T17:20:50.696007465Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T17:20:50.699188553Z","region_metadata_checked_at":"2026-10-24T17:20:50.699188553Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113980401566355456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7741,"pr_state":"closed","pr_sha":"95b7dc25c4967961f7e7ea056397af1375189f54","final_merged_sha":"7c26030ae8e63905ce0f8c90c6f96f0aee339b6b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T13:06:18Z","started_at":"2026-10-24T13:06:18Z","completed_at":"2026-10-24T14:54:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T15:05:13.313225179Z","timing_metadata_checked_at":"2026-10-24T15:05:13.313225179Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T15:05:13.316409367Z","region_metadata_checked_at":"2026-10-24T15:05:13.316409367Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8261dc1cb7e12ae534ad26e1333dcaf9ae6a451025b3415f351bdc9aa6bb65df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113980401566355456","occurred_at":"2026-10-24T13:06:18Z","failed_at":"e2e","signature_id":"cdd4a365bd1f8f5f93c3078adedb43cf9bd9a254480851d78e85c02f3b74fcb3","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2617933fd6e0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40244-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40244-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2617933fd6e0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40244-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:40244-\u003e172.30.0.10:53: read: connection refused\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7734,"pr_state":"closed","pr_sha":"75dc0ba731226e3b4c72c1d052287b982929842f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T12:30:11Z","started_at":"2026-10-24T12:30:11Z","completed_at":"2026-10-24T14:47:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T14:55:01.284387511Z","timing_metadata_checked_at":"2026-10-24T14:55:01.284387511Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T14:55:01.288288999Z","region_metadata_checked_at":"2026-10-24T14:55:01.288288999Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"28169fefce686d76503bb72d2a7103c88824901ff18ca79a4cbae1d7cfca59c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","occurred_at":"2026-10-24T12:30:11Z","failed_at":"e2e","signature_id":"936e85aac84b68ba9d7b02cd2e8d81f0c398e3411296a8498b63a2315e4cba81","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-2v886t\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2ea218b7e820\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x2ea218b7e780\u003e{\n observation: \u003c*errors.errorString | 0x2ea218d2a9b0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x2ea218b7e760\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-2v886t\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2ea218b7e820\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x2ea218b7e780\u003e{\n observation: \u003c*errors.errorString | 0x2ea218d2a9b0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x2ea218b7e760\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b07beb30979513595f32d4d5be3dd4c6554aa41e13d245ba3d8349190e338b46","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"4b846d9011626ac96be80f147d308209a3604e8e71f97de381e1bafbe1fd6773","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","occurred_at":"2026-10-24T12:30:11Z","failed_at":"e2e","signature_id":"a44daeee5e71bfd01ffb050800873ed56d216ca4254ff105a3f77a6cfeb167b0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x11da4f3f5de0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x11da4f3f5de0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"771e7839fd4b33417d10945fc79afffb47c85fe5c6da8e6be6c0b18146a33eb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","occurred_at":"2026-10-24T12:30:11Z","failed_at":"e2e","signature_id":"2560966459f92d4f35ec9c0d4f05ec298c951267691f5f9a53995585f0aca9dd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2tqhb4\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2f8a3019d120\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x2f8a3019d0e0\u003e{\n observation: \u003c*errors.errorString | 0x2f8a325804f0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x2f8a3019cf00\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2tqhb4\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2f8a3019d120\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x2f8a3019d0e0\u003e{\n observation: \u003c*errors.errorString | 0x2f8a325804f0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x2f8a3019cf00\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca3f18ba576fbe4160cedc95bb389561f1f95766f5e0b5c110315741f9671852","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\""}},{"row_id":"839a23ddc9da0c3cdf4d1b908d5fe9a6d967d255c40f4ab393d140042a8b6f81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113971313344253952","occurred_at":"2026-10-24T12:30:11Z","failed_at":"e2e","signature_id":"a8d7c33431d2d5e1c043c56c17b798c04ea45500f5120f8e441080c92ba01b8a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x13fb9bc14400\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x13fb9bc14400\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","b07beb30979513595f32d4d5be3dd4c6554aa41e13d245ba3d8349190e338b46","ca3f18ba576fbe4160cedc95bb389561f1f95766f5e0b5c110315741f9671852"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113966646417690624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7738,"pr_state":"closed","pr_sha":"64a6ad5a1e79178cff522065aa2ef9f0315d5a7a","final_merged_sha":"64a6ad5a1e79178cff522065aa2ef9f0315d5a7a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T12:11:38Z","started_at":"2026-10-24T12:11:38Z","completed_at":"2026-10-24T14:18:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T14:34:37.271854916Z","timing_metadata_checked_at":"2026-10-24T14:34:37.271854916Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T14:34:37.276712101Z","region_metadata_checked_at":"2026-10-24T14:34:37.276712101Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113964216481222656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7741,"pr_state":"closed","pr_sha":"95b7dc25c4967961f7e7ea056397af1375189f54","final_merged_sha":"7c26030ae8e63905ce0f8c90c6f96f0aee339b6b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T12:01:59Z","started_at":"2026-10-24T12:01:59Z","completed_at":"2026-10-24T13:02:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T13:10:37.073789131Z","timing_metadata_checked_at":"2026-10-24T13:10:37.073789131Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T13:10:37.078560117Z","region_metadata_checked_at":"2026-10-24T13:10:37.078560117Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d39b967c5cf4a3a581a06b0674342bebdb0a76f2311cf053aa6f11c6137ca01b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113964216481222656","occurred_at":"2026-10-24T12:01:59Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-24T12:42:08.055Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T12:42:09.448Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j1222656-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T12:42:09.448Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j1222656-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T12:42:10.162Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j1222656-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T12:42:10.162Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j1222656-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T12:42:10.162Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j1222656-svc\ntime=2026-10-24T12:57:12.942Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113940526687326208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7514,"pr_state":"open","pr_sha":"609f7844368495f7af239f668248ce0c1bd49ac8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T10:27:51Z","started_at":"2026-10-24T10:27:51Z","completed_at":"2026-10-24T12:33:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T12:50:26.697278978Z","timing_metadata_checked_at":"2026-10-24T12:50:26.697278978Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T12:50:26.700836566Z","region_metadata_checked_at":"2026-10-24T12:50:26.700836566Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"95d95069074fd342b9f68eb20527a59547c103297447128fa15e58193d668caa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113940526687326208","occurred_at":"2026-10-24T10:27:51Z","failed_at":"e2e","signature_id":"0f394417741593ba503ac249fa660e68b45000e72808dbd8475c79f316f0f9d2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x232a21da8bc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5cnqwcb679kx\" to finish creating: GET https://rp.j7326208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\",\n \"name\": \"eee303a2-7618-4a6c-a91a-e282ee1845c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T11:30:35.210037853Z\",\n \"endTime\": \"2026-10-24T11:49:35.657573237Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y3p5h5t6y6r0a2w-np-1-656n9-dj76m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5cnqwcb679kx--managed/providers/Microsoft.Compute/virtualMachines/y3p5h5t6y6r0a2w-np-1-656n9-dj76m to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y3p5h5t6y6r0a2w-np-1-656n9-dj76m: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-5cnqwcb679kx--managed/y3p5h5t6y6r0a2w-np-1-656n9-dj76m-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5cnqwcb679kx--managed/providers/Microsoft.Network/networkInterfaces/y3p5h5t6y6r0a2w-np-1-656n9-dj76m-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7326208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\n correlation ID: 1204c613-1fa2-4780-b8d4-cf6423a998a8\n client request ID: 70ebbcc0-cca4-424d-8160-3551b5e8f506\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x232a21da8bc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5cnqwcb679kx\" to finish creating: GET https://rp.j7326208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\",\n \"name\": \"eee303a2-7618-4a6c-a91a-e282ee1845c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T11:30:35.210037853Z\",\n \"endTime\": \"2026-10-24T11:49:35.657573237Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y3p5h5t6y6r0a2w-np-1-656n9-dj76m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5cnqwcb679kx--managed/providers/Microsoft.Compute/virtualMachines/y3p5h5t6y6r0a2w-np-1-656n9-dj76m to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y3p5h5t6y6r0a2w-np-1-656n9-dj76m: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-5cnqwcb679kx--managed/y3p5h5t6y6r0a2w-np-1-656n9-dj76m-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5cnqwcb679kx--managed/providers/Microsoft.Network/networkInterfaces/y3p5h5t6y6r0a2w-np-1-656n9-dj76m-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7326208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eee303a2-7618-4a6c-a91a-e282ee1845c4\n correlation ID: 1204c613-1fa2-4780-b8d4-cf6423a998a8\n client request ID: 70ebbcc0-cca4-424d-8160-3551b5e8f506\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910849948094464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7736,"pr_state":"closed","pr_sha":"ed7d121617ab665039646cdf53c36414616c413e","final_merged_sha":"ed7d121617ab665039646cdf53c36414616c413e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T08:29:55Z","started_at":"2026-10-24T08:29:55Z","completed_at":"2026-10-24T08:34:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T08:48:02.565234508Z","timing_metadata_checked_at":"2026-10-24T08:48:02.565234508Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-24T08:48:02.568525194Z","region_metadata_checked_at":"2026-10-24T09:50:54.944798173Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"13da7de6eb61ff7cabe7f747e314da4f881410f75840bea7cef59f9e845cd712","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910849948094464","occurred_at":"2026-10-24T08:29:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7734,"pr_state":"closed","pr_sha":"75dc0ba731226e3b4c72c1d052287b982929842f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T08:29:50Z","started_at":"2026-10-24T08:29:50Z","completed_at":"2026-10-24T10:48:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T11:04:02.090659184Z","timing_metadata_checked_at":"2026-10-24T11:04:02.090659184Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T11:04:02.093600874Z","region_metadata_checked_at":"2026-10-24T11:04:02.093600874Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"0659d625ef14bf52d5e8a5d661d4fd4e4f98fec5e9261371c26584426292821a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","failed_at":"e2e","signature_id":"91906ee05ec48da05bfce86c04cd6b54b0c64ff95e2a58a358219124f81c3325","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-9vhc67\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2f35f4c5c1a0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x2f35f4c5c120\u003e{\n observation: \u003c*errors.errorString | 0x2f35f6f1e830\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x2f35f4c5c0c0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-9vhc67\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2f35f4c5c1a0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x2f35f4c5c120\u003e{\n observation: \u003c*errors.errorString | 0x2f35f6f1e830\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x2f35f4c5c0c0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b07beb30979513595f32d4d5be3dd4c6554aa41e13d245ba3d8349190e338b46","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"2b32daa961feda5742e19571b6e8f69074560547ca2bc2683eccc4ef3d646acd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","failed_at":"e2e","signature_id":"b45772d9c21eeb78596ea8932a4abcae55a029b0f156a15bd528b50bbacb64a2","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x191d86000da0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x191d86000da0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"32d7af12f80927f56a8959cef7f81fb439f3aa320c91f1760005a60bb5c2fc4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","failed_at":"e2e","signature_id":"891240bac405e3baa2932f2cd07a3c04a37450b93782bafba1074f2d0e0d8a3d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-2ll92l\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd8e10b4e180\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0xd8e11f02690\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-2ll92l\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd8e10b4e180\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0xd8e11f02690\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca3f18ba576fbe4160cedc95bb389561f1f95766f5e0b5c110315741f9671852","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\""}},{"row_id":"3656b3e8b992f8b2e1c946e08b98bc5f6248fd7053db75dd0a542bd6f83ab1e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","failed_at":"alert","signature_id":"a845c02ffa47495d53a03c9cf21fb8d837d26d6fb56147af4cab6babc202e39f","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-24T10:00:39Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-24T10:00:39Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-24T10:09:36Z\n Ended: 2026-10-24T10:18:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tl2qrt5450mmm2tpbrvv2e0lvko6ltl\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tl2qrt5450mmm2tpbrvv2e0lvko6ltl/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-24T10:09:36Z\n Ended: 2026-10-24T10:18:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tl2qrt5450mmm2tpbrvv2e0lvko6ltl\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tl2qrt5450mmm2tpbrvv2e0lvko6ltl/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-24T10:10:38Z\n Ended: 2026-10-24T10:21:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tl2qq6lifhtq61o83ao3dcajsdckdi5\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tl2qq6lifhtq61o83ao3dcajsdckdi5/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-24T10:10:38Z\n Ended: 2026-10-24T10:21:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"klusterlet-2tl2qq6lifhtq61o83ao3dcajsdckdi5\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2tl2qq6lifhtq61o83ao3dcajsdckdi5/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"3f8b768c0a17f901f2ba84c6c4d8663a5181536c05d411547f62f29dd6e06ddb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","failed_at":"alert","signature_id":"7de7d944b459e50dfdd29dedf94b6727c8e84d62f3395cbb57514c85a8e7a176","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-24T10:10:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-24T10:10:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.11:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4869376/providers/microsoft.monitor/accounts/services-j4869376\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-kc252\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"96706be577858cea0ac7bc10676c2de8a3fd96207388438dd6cfaff0aa157e45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","failed_at":"e2e","signature_id":"bcc53a189fc6e12ef22dd0c7ca1d120bdf9fb075fbbfca7a88dac32ca4fd6864","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-r4gwvg\"\nUnexpected error:\n \u003c*errors.joinError | 0x32cae7fe4300\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\": nodes \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x32cae7d8f2c0\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\": nodes \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x32cae7d8f2a0\u003e{\n msg: \"failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\": nodes \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x32cae7d8f280\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\": nodes \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x32cae7a38dc0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-5-1-r4gwvg\"\nUnexpected error:\n \u003c*errors.joinError | 0x32cae7fe4300\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\": nodes \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x32cae7d8f2c0\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\": nodes \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x32cae7d8f2a0\u003e{\n msg: \"failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\": nodes \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x32cae7d8f280\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\": nodes \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x32cae7a38dc0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\": nodes \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}},{"row_id":"abdfeb099c05dda5ee339eb63519b3ca867fce903825f1df4d8da1268383ad9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","failed_at":"e2e","signature_id":"95e59c2767f43e6f5ccf12ae6495ea4c7859921f5efc5fc067b0fbd45a7fcefd","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x274951b76fe0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x274951b76fe0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"fcff90402d540a9c5675e2a9d1cf8c3f5d880e9a015bc8a7cc5a227fd73dbece","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910828464869376","occurred_at":"2026-10-24T08:29:50Z","failed_at":"alert","signature_id":"913267547937bb237929f26f32e2f1afb807ffb9a402d1608c34f6a1cfe7cb6c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-24T09:35:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4869376-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-58f4f6fb5b-wj69n\", region=\"centralus\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-58f4f6fb5b-wj69n has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","b07beb30979513595f32d4d5be3dd4c6554aa41e13d245ba3d8349190e338b46","ca3f18ba576fbe4160cedc95bb389561f1f95766f5e0b5c110315741f9671852","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113910790363811840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6772,"pr_state":"closed","pr_sha":"e0afb7920929a2f1a4d48fe5e209a487710d6739","final_merged_sha":"e0afb7920929a2f1a4d48fe5e209a487710d6739","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T08:29:41Z","started_at":"2026-10-24T08:29:41Z","completed_at":"2026-10-24T10:55:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T11:04:01.96037411Z","timing_metadata_checked_at":"2026-10-24T11:04:01.96037411Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T11:04:01.96647099Z","region_metadata_checked_at":"2026-10-24T11:04:01.96647099Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113895350061240320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6191,"pr_state":"open","pr_sha":"0b660a1c1d53873760d8044d183b30ce239eb8fb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T07:28:20Z","started_at":"2026-10-24T07:28:20Z","completed_at":"2026-10-24T10:26:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T10:32:50.51945665Z","timing_metadata_checked_at":"2026-10-24T10:32:50.51945665Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T10:32:50.523157436Z","region_metadata_checked_at":"2026-10-24T10:32:50.523157436Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5592e353d3903ce80d4a6988330a50853d05ae64e5d54b0800064b33deed7174","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113895350061240320","occurred_at":"2026-10-24T07:28:20Z","failed_at":"e2e","signature_id":"c4f2bd800b94fe1f64cf0dcf512f1253dfba262d21febcfcbd9981b5b70eed50","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc346b690c20\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:38043-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:38043-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.001s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc346b690c20\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:38043-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.10:38043-\u003e172.30.0.10:53: read: connection refused\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113878129071099904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7514,"pr_state":"open","pr_sha":"2154e157d0bcfe0043b3382ea02ebc6458a5419d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-24T06:19:54Z","started_at":"2026-10-24T06:19:54Z","completed_at":"2026-10-24T08:19:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T08:26:55.188393224Z","timing_metadata_checked_at":"2026-10-24T08:26:55.188393224Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T08:26:55.19243651Z","region_metadata_checked_at":"2026-10-24T08:26:55.19243651Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850692501245952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T04:30:53Z","started_at":"2026-10-24T04:30:53Z","completed_at":"2026-10-24T06:59:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T07:12:58.979458558Z","timing_metadata_checked_at":"2026-10-24T07:12:58.979458558Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T07:12:58.982930448Z","region_metadata_checked_at":"2026-10-24T07:12:58.982930448Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"35c16c3136532753f311d95ca7284cb89d2efa1eafc4df4f6e5319299ab1f658","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850692501245952","occurred_at":"2026-10-24T04:30:53Z","failed_at":"e2e","signature_id":"2437bca1dfdab8003e0f75fb95e845e5a73877ed5b1c32672694464dd4e2adda","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 601.097s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x11218fd9090\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 601.097s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x11218fd9090\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20d58027fbe6fa8b683210764894d620c01183ffc035b6c390592701e4ed6fda","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11200, acm-agent=0"}},{"row_id":"e479891268631ca4e4e4fa411d3f0c38d3921fade44658aeb6714b2003ac2c4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850692501245952","occurred_at":"2026-10-24T04:30:53Z","failed_at":"alert","signature_id":"4092fa71e6585c9f0469827fb61244698d5b97435db774acf42bc2d72d8bb3a3","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-24T05:47:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1245952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1245952/providers/microsoft.monitor/accounts/services-j1245952\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-s9thv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ff2d3a94-ed8f-48bd-b318-c937cce68e20\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-s9thv (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-24T05:47:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1245952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1245952/providers/microsoft.monitor/accounts/services-j1245952\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-s9thv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ff2d3a94-ed8f-48bd-b318-c937cce68e20\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-s9thv (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-24T05:49:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1245952-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.30:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1245952/providers/microsoft.monitor/accounts/services-j1245952\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-lpxww\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"346ba3f6-80dd-4a56-add6-18f9140c124e\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-lpxww (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-24T05:51:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1245952-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.30:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1245952/providers/microsoft.monitor/accounts/services-j1245952\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-lpxww\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"346ba3f6-80dd-4a56-add6-18f9140c124e\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-lpxww (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["20d58027fbe6fa8b683210764894d620c01183ffc035b6c390592701e4ed6fda","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850684494319616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6772,"pr_state":"closed","pr_sha":"e0afb7920929a2f1a4d48fe5e209a487710d6739","final_merged_sha":"e0afb7920929a2f1a4d48fe5e209a487710d6739","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T04:30:51Z","started_at":"2026-10-24T04:30:51Z","completed_at":"2026-10-24T06:55:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T07:02:50.24251787Z","timing_metadata_checked_at":"2026-10-24T07:02:50.24251787Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T07:02:50.246249658Z","region_metadata_checked_at":"2026-10-24T07:02:50.246249658Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"efeb747805f5ee630ce3ffa283be2347ed92e6eca0fb480b325f808ccd962740","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850684494319616","occurred_at":"2026-10-24T04:30:51Z","failed_at":"e2e","signature_id":"5577b5c675e94c0cb99a9451504aa28e42c87218c9dfcaaed681e5e942070b2b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x286c46f305a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x286c46f30580\u003e{\n observation: \u003c*errors.errorString | 0x286c458cdbb0\u003e{\n s: \"dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x286c46f30560\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T05:33:22.4488158Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x286c4628c1e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x286c46f30540\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T05:33:22.4488158Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x286c478a50e0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x286c46f305a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x286c46f30580\u003e{\n observation: \u003c*errors.errorString | 0x286c458cdbb0\u003e{\n s: \"dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-2tkvarldubqmpnbonti2lc7tvjkd8h7v-m2m5f8i0z1w3d9n: expected all current routers \u003e= desired replicas \u003e 0; desired=0 current=0 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x286c46f30560\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T05:33:22.4488158Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x286c4628c1e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x286c46f30540\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-wkh9kwhf6mgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T05:33:22.4488158Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x286c478a50e0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a6e167b4896a6cc8e5dc9db9cc3f6643b41874460ed57be902ccc03e9ff1333","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j4319616-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8a6e167b4896a6cc8e5dc9db9cc3f6643b41874460ed57be902ccc03e9ff1333"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7734,"pr_state":"closed","pr_sha":"75dc0ba731226e3b4c72c1d052287b982929842f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T04:30:50Z","started_at":"2026-10-24T04:30:50Z","completed_at":"2026-10-24T06:53:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T07:02:50.263798902Z","timing_metadata_checked_at":"2026-10-24T07:02:50.263798902Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T07:02:50.266623393Z","region_metadata_checked_at":"2026-10-24T07:02:50.266623393Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"2c5675384926910724de0ad726502e380105f8f71c99b8914fa7263e7f8f94d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","failed_at":"e2e","signature_id":"93d4f78b327358bdeb0108f6eae0945f79cb06676ae17e59a0390dca843aefea","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f6cce913900\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b559ft\" in resourcegroup=\"rg-np-version-upgrade-b559ft-j4qdwfs6szxx\" to finish creating: GET https://rp.j3761152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\",\n \"name\": \"489dec27-e4f4-49eb-a330-6052b204ad8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T05:53:26.952860579Z\",\n \"endTime\": \"2026-10-24T06:12:28.455904126Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b559ft-j4qdwfs6szxx-np-upgrade-b5-5fe002ad/providers/Microsoft.Compute/virtualMachines/p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b559ft-j4qdwfs6szxx-np-upgrade-b5-5fe002ad/providers/Microsoft.Compute/virtualMachines/p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3761152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\n correlation ID: d5af14b9-8e48-4974-af98-abb8c215a6d4\n client request ID: 1283d9b2-c265-4f0a-97ea-52c82d8cd670\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n \u003c*fmt.wrapError | 0x1f6cce913900\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b559ft\" in resourcegroup=\"rg-np-version-upgrade-b559ft-j4qdwfs6szxx\" to finish creating: GET https://rp.j3761152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\",\n \"name\": \"489dec27-e4f4-49eb-a330-6052b204ad8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T05:53:26.952860579Z\",\n \"endTime\": \"2026-10-24T06:12:28.455904126Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b559ft-j4qdwfs6szxx-np-upgrade-b5-5fe002ad/providers/Microsoft.Compute/virtualMachines/p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b559ft-j4qdwfs6szxx-np-upgrade-b5-5fe002ad/providers/Microsoft.Compute/virtualMachines/p8m7i5l8y6g1n1v-npupgrade-4-20-kdgcn-qg9p4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3761152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/489dec27-e4f4-49eb-a330-6052b204ad8e\n correlation ID: d5af14b9-8e48-4974-af98-abb8c215a6d4\n client request ID: 1283d9b2-c265-4f0a-97ea-52c82d8cd670\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"625fd9194aaf1b3e71d69ebff39c4c875ca8e2aff5204fa93648ccf6902233d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","failed_at":"e2e","signature_id":"b84151051ec5b2adf25427fb8178337ecae54b7aa256dc8f766dcc6c6a454410","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3d4f883a52d0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3d4f883a52d0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"80518fd13efd3e2e4e6eb9df4bc7aaf59383cc65696273f01ed1767d4d326024","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","failed_at":"e2e","signature_id":"9b0fa023f70c43f9bd584cf2b92c6a19d34fe34c5b3a361cf4f12f49592982d4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-8db7r8\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x31b88a1146c0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x31b88a1146a0\u003e{\n observation: \u003c*errors.errorString | 0x31b88a000c30\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x31b88a114640\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-8db7r8\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x31b88a1146c0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x31b88a1146a0\u003e{\n observation: \u003c*errors.errorString | 0x31b88a000c30\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x31b88a114640\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b07beb30979513595f32d4d5be3dd4c6554aa41e13d245ba3d8349190e338b46","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"9942e38a6e0db84ef6e9bfda477aff1cb29bb62eaf54c6c1defc33804a15c918","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","failed_at":"e2e","signature_id":"1c556c68dc0e81b592bd96ea684cc1bf9b6af1fb39f1b4904e1224a3d612c036","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-84tw8j\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x347780605600\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x3477806055c0\u003e{\n observation: \u003c*errors.errorString | 0x34778111e160\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x3477806055a0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-84tw8j\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x347780605600\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x3477806055c0\u003e{\n observation: \u003c*errors.errorString | 0x34778111e160\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x3477806055a0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b07beb30979513595f32d4d5be3dd4c6554aa41e13d245ba3d8349190e338b46","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"b69e74f378dc702d80f95b957f967db08339462722140fc057207c31044d7a4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850679603761152","occurred_at":"2026-10-24T04:30:50Z","failed_at":"e2e","signature_id":"5a97ec00b2365f6302a91eeb09a77807c49fff76d3d016fc94f23dcb1ee35688","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x9365e5f86a0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x9365e5f86a0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","b07beb30979513595f32d4d5be3dd4c6554aa41e13d245ba3d8349190e338b46","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850601719730176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7736,"pr_state":"closed","pr_sha":"ed7d121617ab665039646cdf53c36414616c413e","final_merged_sha":"ed7d121617ab665039646cdf53c36414616c413e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T04:30:31Z","started_at":"2026-10-24T04:30:31Z","completed_at":"2026-10-24T06:43:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T06:51:58.784847706Z","timing_metadata_checked_at":"2026-10-24T06:51:58.784847706Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T06:51:58.789962486Z","region_metadata_checked_at":"2026-10-24T06:51:58.789962486Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"188dc1f8fcfffcec16d46ab91b66a6a7a2f4ad08a482233e0dd8cede2f0840f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113850601719730176","occurred_at":"2026-10-24T04:30:31Z","failed_at":"e2e","signature_id":"73e414d801e26a3b9d8e077db7186e53205ae2c6f4de6a0577eb3a964c24a61f","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x298d72d44040\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-gw598c\" in resourcegroup=\"rg-np-noedge-gw598c-2zsqpwzqbmb8\" to finish creating: GET https://rp.j9730176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea64491d-d0fd-400e-88eb-8c8bd1cb256b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea64491d-d0fd-400e-88eb-8c8bd1cb256b\",\n \"name\": \"ea64491d-d0fd-400e-88eb-8c8bd1cb256b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T05:54:16.071407678Z\",\n \"endTime\": \"2026-10-24T06:13:22.793053391Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u8s0r4g9w5s8j0u-npnoedge-4-20-q9dz8-xgzd2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-gw598c-2zsqpwzqbmb8-np-ne-gw598c--managed/providers/Microsoft.Compute/virtualMachines/u8s0r4g9w5s8j0u-npnoedge-4-20-q9dz8-xgzd2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u8s0r4g9w5s8j0u-npnoedge-4-20-q9dz8-xgzd2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-gw598c-2zsqpwzqbmb8-np-ne-gw598c--managed/providers/Microsoft.Compute/virtualMachines/u8s0r4g9w5s8j0u-npnoedge-4-20-q9dz8-xgzd2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9730176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea64491d-d0fd-400e-88eb-8c8bd1cb256b\n correlation ID: 149b615d-592f-486c-b354-94d297e4fa6d\n client request ID: eb91eb91-fd3e-493e-b389-b15c2dca94ef\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x298d72d44040\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-gw598c\" in resourcegroup=\"rg-np-noedge-gw598c-2zsqpwzqbmb8\" to finish creating: GET https://rp.j9730176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea64491d-d0fd-400e-88eb-8c8bd1cb256b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea64491d-d0fd-400e-88eb-8c8bd1cb256b\",\n \"name\": \"ea64491d-d0fd-400e-88eb-8c8bd1cb256b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T05:54:16.071407678Z\",\n \"endTime\": \"2026-10-24T06:13:22.793053391Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u8s0r4g9w5s8j0u-npnoedge-4-20-q9dz8-xgzd2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-gw598c-2zsqpwzqbmb8-np-ne-gw598c--managed/providers/Microsoft.Compute/virtualMachines/u8s0r4g9w5s8j0u-npnoedge-4-20-q9dz8-xgzd2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u8s0r4g9w5s8j0u-npnoedge-4-20-q9dz8-xgzd2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-gw598c-2zsqpwzqbmb8-np-ne-gw598c--managed/providers/Microsoft.Compute/virtualMachines/u8s0r4g9w5s8j0u-npnoedge-4-20-q9dz8-xgzd2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9730176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea64491d-d0fd-400e-88eb-8c8bd1cb256b\n correlation ID: 149b615d-592f-486c-b354-94d297e4fa6d\n client request ID: eb91eb91-fd3e-493e-b389-b15c2dca94ef\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113827015499976704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7514,"pr_state":"open","pr_sha":"70ec6be3b30a7731ef7d05f42593e4dd6d7a8023","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T02:56:48Z","started_at":"2026-10-24T02:56:48Z","completed_at":"2026-10-24T04:54:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T05:07:11.506962703Z","timing_metadata_checked_at":"2026-10-24T05:07:11.506962703Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T05:07:11.510318688Z","region_metadata_checked_at":"2026-10-24T05:07:11.510318688Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"33ddd636cd4df1c262ecbca83bed0b3f999409e72c017174c1dcefecc0f346f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113827015499976704","occurred_at":"2026-10-24T02:56:48Z","failed_at":"e2e","signature_id":"8c262f00ec05d49570da12d0f71a5cb18b3aa7c5602d40ca3360b5c689bea7bb","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.002s.\ncert-manager deployment should complete rollout with workload identity\nExpected success, but got an error:\n \u003c*errors.errorString | 0x224df5000400\u003e: \n cert-manager rollout in progress: updated=1 ready=1 desired=2\n {\n s: \"cert-manager rollout in progress: updated=1 ready=1 desired=2\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.002s.\ncert-manager deployment should complete rollout with workload identity\nExpected success, but got an error:\n \u003c*errors.errorString | 0x224df5000400\u003e: \n cert-manager rollout in progress: updated=1 ready=1 desired=2\n {\n s: \"cert-manager rollout in progress: updated=1 ready=1 desired=2\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed62f5f6e15089a288745d0cb27e1873c68853958b7d80abb95ff11fd25b0be8","failure_pattern":"cert-manager rollout in progress: updated=1 ready=1 desired=2","search_query":"cert-manager rollout in progress: updated=1 ready=1 desired=2"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ed62f5f6e15089a288745d0cb27e1873c68853958b7d80abb95ff11fd25b0be8"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113821515370729472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7734,"pr_state":"closed","pr_sha":"75dc0ba731226e3b4c72c1d052287b982929842f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T02:34:56Z","started_at":"2026-10-24T02:34:56Z","completed_at":"2026-10-24T03:41:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:53:37.810171076Z","timing_metadata_checked_at":"2026-10-24T03:53:37.810171076Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:53:37.813671066Z","region_metadata_checked_at":"2026-10-24T03:53:37.813671066Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c80600f1bfc205d740551db3f45fc3112e095672e638ccd68962a38b1a30afbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7734/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113821515370729472","occurred_at":"2026-10-24T02:34:56Z","failed_at":"provision","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"templatize-pipeline","failure_text":"time=2026-10-24T03:19:41.940Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T03:19:44.124Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T03:19:44.124Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T03:19:45.250Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T03:19:45.250Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:19:45.250Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j0729472-svc\ntime=2026-10-24T03:21:51.429Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-24T03:21:51.429Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m6s\ntime=2026-10-24T03:21:51.429Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:51.773Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-24T03:21:51.773Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:51.773Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:52.375Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-24T03:21:52.375Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:52.438Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-24T03:21:52.730Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:52.857Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j0729472-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T03:21:53.744Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j0729472-svc\ntime=2026-10-24T03:36:56.654Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113819560179142656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7736,"pr_state":"closed","pr_sha":"ed7d121617ab665039646cdf53c36414616c413e","final_merged_sha":"ed7d121617ab665039646cdf53c36414616c413e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T02:27:09Z","started_at":"2026-10-24T02:27:09Z","completed_at":"2026-10-24T02:58:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:12:43.086590263Z","timing_metadata_checked_at":"2026-10-24T03:12:43.086590263Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-24T03:12:43.089650954Z","region_metadata_checked_at":"2026-10-24T04:14:47.660595327Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d4c927be992d06356c24181c274bdec891d6d121392df75d84f732418c1e0990","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113819560179142656","occurred_at":"2026-10-24T02:27:09Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113797768924368896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6081,"pr_state":"open","pr_sha":"860fe9f16ea44af6ed07ffbd4b0e9c062b01f3d7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T01:00:13Z","started_at":"2026-10-24T01:00:13Z","completed_at":"2026-10-24T03:17:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:33:17.15814046Z","timing_metadata_checked_at":"2026-10-24T03:33:17.15814046Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:33:17.161431751Z","region_metadata_checked_at":"2026-10-24T03:33:17.161431751Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a5ce3c5ca36659f07415f5420390edf6503dfe69bd977e553715ebb39f9fe108","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6081/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113797768924368896","occurred_at":"2026-10-24T01:00:13Z","failed_at":"alert","signature_id":"6ef254c858e87304545e34d73618328753d7ee1ecc5a887748fef0be19e5e856","test_name":"[aro-hcp-observability] [svc] alert ClusterCredentialMetricMissing does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-24T02:11:05Z\n Severity: Sev3\n Labels: alertname=\"ClusterCredentialMetricMissing\", cluster=\"ci01-j4368896-svc\", severity=\"warning\"\n Description: Credential refresher expiration metric is missing for service cluster ci01-j4368896-svc.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583","failure_pattern":"alert [svc] ClusterCredentialMetricMissing fired","search_query":"alert [svc] ClusterCredentialMetricMissing fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e1d4685909dc768faa8a74d046911abd324b4f49a6b9ba66534225bdfde20583"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113797113543397376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6580,"pr_state":"open","pr_sha":"1fe9af59ce06c5d6f785a91421edf88db61a778f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T00:57:58Z","started_at":"2026-10-24T00:57:58Z","completed_at":"2026-10-24T03:11:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:23:13.003101773Z","timing_metadata_checked_at":"2026-10-24T03:23:13.003101773Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:23:13.007441761Z","region_metadata_checked_at":"2026-10-24T03:23:13.007441761Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cef481eccef7b3403966d92f93c7ab635ae4c7368f643d2b14426f078aa10fa7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113797113543397376","occurred_at":"2026-10-24T00:57:58Z","failed_at":"e2e","signature_id":"d8f417e401cde35fdf1ac72973ebbf582471bdc57ba9ddd78dbb706c8363d414","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-xz7cz7\"\nUnexpected error:\n \u003c*errors.joinError | 0x251e30c232d8\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\": nodes \"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x251e2f894e60\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\": nodes \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x251e2f894de0\u003e{\n msg: \"failed to label node r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\": nodes \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x251e2f894dc0\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\": nodes \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x251e2fba9180\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-5-0-xz7cz7\"\nUnexpected error:\n \u003c*errors.joinError | 0x251e30c232d8\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\": nodes \"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x251e2f894e60\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\": nodes \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x251e2f894de0\u003e{\n msg: \"failed to label node r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\": nodes \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x251e2f894dc0\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\": nodes \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x251e2fba9180\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"r3d9q3y4d1r6m9l-np-xz7cz7-4flsb-cc9qc\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\": nodes \"l8x3u9g8g1l7q7r-np-r4gwvg-vbqhr-7vz2h\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113796816469233664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7738,"pr_state":"closed","pr_sha":"64a6ad5a1e79178cff522065aa2ef9f0315d5a7a","final_merged_sha":"64a6ad5a1e79178cff522065aa2ef9f0315d5a7a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T00:56:48Z","started_at":"2026-10-24T00:56:48Z","completed_at":"2026-10-24T03:12:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:23:14.396111055Z","timing_metadata_checked_at":"2026-10-24T03:23:14.396111055Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:23:14.398946748Z","region_metadata_checked_at":"2026-10-24T03:23:14.398946748Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3d2e8daedd4127ff38130f102ddd80fe01220b7c64a6b4023864bfb783e2388b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113796816469233664","occurred_at":"2026-10-24T00:56:48Z","failed_at":"e2e","signature_id":"2782ca05257d433f2f9cf504b4ee91198fa29b5e3ce21b6b86156326f1b22374","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x20a8e1c0eb40\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x20a8e1c0eb20\u003e{\n observation: \u003c*errors.errorString | 0x20a8e1fc7a50\u003e{\n s: \"dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x20a8e1c0eb00\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T01:52:41.1693145Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x20a8e189aaf0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x20a8e1c0eae0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T01:52:41.1693145Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x20a8e31d5b60\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x20a8e1c0eb40\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x20a8e1c0eb20\u003e{\n observation: \u003c*errors.errorString | 0x20a8e1fc7a50\u003e{\n s: \"dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-2tks36bjou1ubb8nv8hnufkiqucajqrv-e4f0v9j4v7w6l2h: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x20a8e1c0eb00\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T01:52:41.1693145Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x20a8e189aaf0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x20a8e1c0eae0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-9qlmbzgtx2hq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-24T01:52:41.1693145Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x20a8e31d5b60\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d2678f56dc1b3b431d1da623f447c6eafacda9e82db7dae4d918a344f1c056b7","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9233664-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d2678f56dc1b3b431d1da623f447c6eafacda9e82db7dae4d918a344f1c056b7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113796458485387264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7730,"pr_state":"open","pr_sha":"9b2b97213e4ec0944e8280a5b0e47de5993822fc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T00:55:22Z","started_at":"2026-10-24T00:55:22Z","completed_at":"2026-10-24T03:10:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:23:13.837773886Z","timing_metadata_checked_at":"2026-10-24T03:23:13.837773886Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:23:13.840865777Z","region_metadata_checked_at":"2026-10-24T03:23:13.840865777Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0f1bcb4fe118df37d4bff9b75233ec54002ca44526676a01d8eb3d66f300bd77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113796458485387264","occurred_at":"2026-10-24T00:55:22Z","failed_at":"e2e","signature_id":"2c4af7aec4f24afaa73fdc991afb8658d4ba85d9f78d068284914a918c56028a","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x19f71c1da380\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-frlx5lsjl64c\" to finish creating: GET https://rp.j5387264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb737a8-b808-43b4-bafc-d691d15a4945\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb737a8-b808-43b4-bafc-d691d15a4945\",\n \"name\": \"6eb737a8-b808-43b4-bafc-d691d15a4945\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T02:13:35.7654536Z\",\n \"endTime\": \"2026-10-24T02:32:39.313667809Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y9f1n7k6s3b3s7k-np-deletable-csm74-d5dxn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-frlx5lsjl64c--managed/providers/Microsoft.Compute/virtualMachines/y9f1n7k6s3b3s7k-np-deletable-csm74-d5dxn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y9f1n7k6s3b3s7k-np-deletable-csm74-d5dxn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-frlx5lsjl64c--managed/providers/Microsoft.Compute/virtualMachines/y9f1n7k6s3b3s7k-np-deletable-csm74-d5dxn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5387264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb737a8-b808-43b4-bafc-d691d15a4945\n correlation ID: b38b110a-9400-47b0-9d30-29c40b6745e7\n client request ID: 160823fa-0861-4cd5-b970-f7ba5cee199c\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x19f71c1da380\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-frlx5lsjl64c\" to finish creating: GET https://rp.j5387264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb737a8-b808-43b4-bafc-d691d15a4945\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb737a8-b808-43b4-bafc-d691d15a4945\",\n \"name\": \"6eb737a8-b808-43b4-bafc-d691d15a4945\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T02:13:35.7654536Z\",\n \"endTime\": \"2026-10-24T02:32:39.313667809Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y9f1n7k6s3b3s7k-np-deletable-csm74-d5dxn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-frlx5lsjl64c--managed/providers/Microsoft.Compute/virtualMachines/y9f1n7k6s3b3s7k-np-deletable-csm74-d5dxn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y9f1n7k6s3b3s7k-np-deletable-csm74-d5dxn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-frlx5lsjl64c--managed/providers/Microsoft.Compute/virtualMachines/y9f1n7k6s3b3s7k-np-deletable-csm74-d5dxn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5387264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6eb737a8-b808-43b4-bafc-d691d15a4945\n correlation ID: b38b110a-9400-47b0-9d30-29c40b6745e7\n client request ID: 160823fa-0861-4cd5-b970-f7ba5cee199c","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-frlx5lsjl64c\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113795606852931584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T00:51:59Z","started_at":"2026-10-24T00:51:59Z","completed_at":"2026-10-24T03:00:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:12:42.922577457Z","timing_metadata_checked_at":"2026-10-24T03:12:42.922577457Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:12:42.925748548Z","region_metadata_checked_at":"2026-10-24T03:12:42.925748548Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0b8f3c40a9304a5fc2be63ecabb25d30b7ee1c7e9351ddd039cdb5fee7f02748","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113795606852931584","occurred_at":"2026-10-24T00:51:59Z","failed_at":"e2e","signature_id":"2478f6179eaff1c358edd52ab231bf86c77990919d3c9e804395e4e4562a42aa","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2de7945ac470\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.009s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2de7945ac470\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b727ea5a9ba2a3858b4292ab83d8c90a401379a8446650e11bad43c6ae5ed290","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1491, hypershift=11519, acm-agent=0"}},{"row_id":"8485c03c9f6bef07e8cd8b1cb263249dddc822e45482551948e509a432b581ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113795606852931584","occurred_at":"2026-10-24T00:51:59Z","failed_at":"alert","signature_id":"7fe27bdbd1b619b6b49df4650d8476278c6f6132c533ba3ef57f974fe7697567","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-24T02:00:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2931584-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.79:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2931584/providers/microsoft.monitor/accounts/services-j2931584\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-kftnj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"71b5d22f-cbab-4867-b020-104124d57ca6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-kftnj (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-24T02:01:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2931584-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2931584/providers/microsoft.monitor/accounts/services-j2931584\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-w7rhw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a1d67f6c-7471-4c26-92ba-168a6c18274b\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-w7rhw (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-24T02:03:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2931584-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.79:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2931584/providers/microsoft.monitor/accounts/services-j2931584\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-kftnj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"71b5d22f-cbab-4867-b020-104124d57ca6\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-kftnj (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-24T02:03:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2931584-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2931584/providers/microsoft.monitor/accounts/services-j2931584\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-w7rhw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a1d67f6c-7471-4c26-92ba-168a6c18274b\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-w7rhw (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","b727ea5a9ba2a3858b4292ab83d8c90a401379a8446650e11bad43c6ae5ed290"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113795531460317184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7120,"pr_state":"closed","pr_sha":"2e62dfb44ee3a1d443eb626604f2e79321982d6b","final_merged_sha":"2e62dfb44ee3a1d443eb626604f2e79321982d6b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T00:51:41Z","started_at":"2026-10-24T00:51:41Z","completed_at":"2026-10-24T03:05:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:23:13.473729583Z","timing_metadata_checked_at":"2026-10-24T03:23:13.473729583Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:23:13.477023774Z","region_metadata_checked_at":"2026-10-24T03:23:13.477023774Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113794936213082112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6191,"pr_state":"open","pr_sha":"0b660a1c1d53873760d8044d183b30ce239eb8fb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T00:49:19Z","started_at":"2026-10-24T00:49:19Z","completed_at":"2026-10-24T03:33:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:43:22.171212881Z","timing_metadata_checked_at":"2026-10-24T03:43:22.171212881Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:43:22.175689963Z","region_metadata_checked_at":"2026-10-24T03:43:22.175689963Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b057649411d5d0f6ee592107f6ff415cfad4babe6bcab512662aae0ae310eff4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113794936213082112","occurred_at":"2026-10-24T00:49:19Z","failed_at":"e2e","signature_id":"d0567201314e2f51df5880f0f1f4bfed0b2ec9921b0de43f8e38feddc4717086","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xda86742c750\u003e: \n VerifyNodesReady failed: there are not ready nodes: cilium-cluster-cilium-np-kph42-vfrqk (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-24T02:36:22Z)\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fwbfcwb579g7\" to finish creating: GET https://rp.j3082112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\",\n \"name\": \"40c4c34c-751c-413b-8e7b-4d8aef1fbbba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T02:18:41.838217414Z\",\n \"endTime\": \"2026-10-24T02:37:46.632619034Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-kph42-vfrqk: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-kph42-vfrqk: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3082112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\n correlation ID: 260f62ef-02cf-4e9c-83e2-0f2095de0a22\n client request ID: 0ea1151f-ae49-4a4c-83b3-a2443bb08376\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xda86742c750\u003e: \n VerifyNodesReady failed: there are not ready nodes: cilium-cluster-cilium-np-kph42-vfrqk (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-24T02:36:22Z)\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fwbfcwb579g7\" to finish creating: GET https://rp.j3082112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\",\n \"name\": \"40c4c34c-751c-413b-8e7b-4d8aef1fbbba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T02:18:41.838217414Z\",\n \"endTime\": \"2026-10-24T02:37:46.632619034Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-kph42-vfrqk: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-kph42-vfrqk: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3082112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40c4c34c-751c-413b-8e7b-4d8aef1fbbba\n correlation ID: 260f62ef-02cf-4e9c-83e2-0f2095de0a22\n client request ID: 0ea1151f-ae49-4a4c-83b3-a2443bb08376\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113793086860562432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6772,"pr_state":"closed","pr_sha":"e0afb7920929a2f1a4d48fe5e209a487710d6739","final_merged_sha":"e0afb7920929a2f1a4d48fe5e209a487710d6739","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T00:41:58Z","started_at":"2026-10-24T00:41:58Z","completed_at":"2026-10-24T02:56:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:12:42.697811035Z","timing_metadata_checked_at":"2026-10-24T03:12:42.697811035Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:12:42.703750217Z","region_metadata_checked_at":"2026-10-24T03:12:42.703750217Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7a474033528a495d8b3e073706e3717eabb8b4b1d2009a84ff175a97f691ce15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113793086860562432","occurred_at":"2026-10-24T00:41:58Z","failed_at":"alert","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-24T01:56:07Z\n Ended: 2026-10-24T02:15:08Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0562432-svc\", component=\"backend\", name=\"csstatedump\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-24T01:57:06Z\n Ended: 2026-10-24T02:15:08Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0562432-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6173/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113792299900080128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6173,"pr_state":"open","pr_sha":"47739c8f709710ccb1c4ef21f3dbe8fde0644d6f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-24T00:38:51Z","started_at":"2026-10-24T00:38:51Z","completed_at":"2026-10-24T03:17:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T03:33:17.278006506Z","timing_metadata_checked_at":"2026-10-24T03:33:17.278006506Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T03:33:17.281169897Z","region_metadata_checked_at":"2026-10-24T03:33:17.281169897Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113789146521669632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-24T00:26:19Z","started_at":"2026-10-24T00:26:19Z","completed_at":"2026-10-24T00:45:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:48:12.56622351Z","timing_metadata_checked_at":"2026-10-24T00:48:12.56622351Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-24T00:48:12.568982006Z","region_metadata_checked_at":"2026-10-24T01:50:04.357021154Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"54ae4755f28ec63281af3a546ede2b894e5628f23c3f51fbf342c831fb16bd1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113789146521669632","occurred_at":"2026-10-24T00:26:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113783035924058112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7371,"pr_state":"closed","pr_sha":"42b177e82e37e27d732edd9dee43c759e521c0b1","final_merged_sha":"42b177e82e37e27d732edd9dee43c759e521c0b1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-24T00:01:20Z","started_at":"2026-10-24T00:01:20Z","completed_at":"2026-10-24T02:37:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T02:51:43.925157626Z","timing_metadata_checked_at":"2026-10-24T02:51:43.925157626Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T02:51:43.92962141Z","region_metadata_checked_at":"2026-10-24T02:51:43.92962141Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}