{"meta":{"date":"2026-10-23","timezone":"UTC","generated_at":"2026-10-28T23:52:17Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":14,"failed_runs":7,"runs_with_occurrences":7,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":7,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T23:38:10Z","started_at":"2026-10-23T23:38:10Z","completed_at":"2026-10-24T01:52:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T02:10:48.4346559Z","timing_metadata_checked_at":"2026-10-24T02:10:48.4346559Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T02:10:48.437841691Z","region_metadata_checked_at":"2026-10-24T02:10:48.437841691Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1482942acb8bddb32e344a714ab97b5a2aa73da75f224eb866015beb2d663dc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"959d7136fa8fa3534927661a3d60094a7592b6d059d009358f71e15228fe76e2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0x1a3727108180\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1a37266c48a0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1a3726894020\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1a3725d94d20\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0x1a3727108180\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1a37266c48a0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1a3726894020\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1a3725d94d20\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"509ce29f121aa8889071be97ef0ea23faf908d0bfad41dc19a0232af98d72839","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"1f69fb4068de7e74d39c72d203356884025874f6e166da3163cb8d4ebde2b56f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"fffc6a8a7c348c164995910dd1c108865b80e864e9a58f247d4be169e8a399d2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-5ltfgj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c3759b85ac0\u003e: \n DNS for route host agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1c3759b840a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-5ltfgj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c3759b85ac0\u003e: \n DNS for route host agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1c3759b840a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dq8j6.apps.aro.s6s7c3r4z7y9i8e.gjrq.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"2a3a567a3b9c26cedc32587846ba66db3578f32209f4f028df46560a2b537ebd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"eeff509555b3bd3bab3c3359799b2c2e597d7c59acb0e88f24bfbf61670e37e3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-5p9hrv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x359991ca6200\u003e: \n DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x359991cc83e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-5p9hrv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x359991ca6200\u003e: \n DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x359991cc83e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dbdwm.apps.aro.n0p5j9k4k3x6p4o.lcm9.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"2a76824bf9c9c4c9b1f66a24b36486c06f285e7dfdd7ad82dc9dcbc6e48c8deb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"a23174899568b6c480cde29c16f107e2df7fb592b2576c4598c4dd3b3ed64763","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-wnz6kv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1aca60620e60\u003e: \n DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1aca604dcae0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-wnz6kv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1aca60620e60\u003e: \n DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1aca604dcae0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5cwbl.apps.aro.c1s1r7e8q7f4j2u.vfmi.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"4989d1da555472792f76633cc7aaa3813eed80fdff11c30eb73d934ccecbdeef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"9dfcee82299c274c6cbfd06b3f9fb3c7b36ad00e636501927c19be2e530212db","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x2d202c117968\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x2d202bb0ece0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2d202ad36960\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2d202a99c2c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x2d202c117968\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x2d202bb0ece0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2d202ad36960\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2d202a99c2c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ec625aa187990188920942c47358d3eb1c7eda37bdf66a6d43c4b3578a43526","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sv2qn.apps.aro.cilium-cl.43f6.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"53aec1470e0bff138f43fad24ca16c14bee6e22ac0e92670a3e576ca5b7372e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"0b5d6aa96dc387e24884d930781d5cd7d6557a061c28ab9733a3d3b1c2b4c58a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-vw82ff\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x13ad1fd46ea0\u003e: \n DNS for route host agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13ad1fb24980\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-1-vw82ff\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x13ad1fd46ea0\u003e: \n DNS for route host agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x13ad1fb24980\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-fxqbn.apps.aro.c7f7b4c1b4z0q6b.dcr4.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"6423a55660e1fcc4b8df41c9e242414c0d3812a5460b4ec793c9b7b1e9de894b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"86a169f78b189ca410e85207ae5eedf317f888d81a7fbf1f8ae05a6c40c5927e","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n \u003c*url.Error | 0x20ec8dc8cd20\u003e: \n Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n {\n Op: \"Get\",\n URL: \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n Err: \u003c*net.OpError | 0x20ec8bc10690\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0x20ec8bc103c0\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:225]: Timed out after 600.000s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:223 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\nUnexpected error:\n \u003c*url.Error | 0x20ec8dc8cd20\u003e: \n Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host\n {\n Op: \"Get\",\n URL: \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n Err: \u003c*net.OpError | 0x20ec8bc10690\u003e{\n Op: \"dial\",\n Net: \"tcp\",\n Source: nil,\n Addr: nil,\n Err: \u003c*net.DNSError | 0x20ec8bc103c0\u003e{\n UnwrapErr: nil,\n Err: \"no such host\",\n Name: \"console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\",\n Server: \"172.30.0.10:53\",\n IsTimeout: false,\n IsTemporary: false,\n IsNotFound: true,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3ce2ba89b8e9686083a83dadac7501168f4efe9a7eacff619946e50e9db1c762","failure_pattern":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.w1c2.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host"}},{"row_id":"767c1b8e90977a0f557a0911a4bdb839a72017f02b5edf33405c0230b118b2fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"103697c5a8db15b41fc2a08d46ab15e59d6e4653e09a4e1f747b198e64c261d6","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x3c147812a9c0\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x3c1477af35c0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3c14791742c0\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3c14779bc3c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:408]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x3c147812a9c0\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x3c1477af35c0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3c14791742c0\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3c14779bc3c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j4nk2.apps.aro.agg-cluster.as13.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"509ce29f121aa8889071be97ef0ea23faf908d0bfad41dc19a0232af98d72839","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"8f6648e526aeeca972310ec1c4851eccb56ad2aa84ca092bc290ea55f758e9b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"05baccf373c01bd6f7562dcbc3236b5780751cf9ec7eae7616fd70c9c8c3da1a","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xdf1d3a48c78\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0xdf1d48b58c0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xdf1d48b46a0\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xdf1d394a3a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xdf1d3a48c78\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0xdf1d48b58c0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xdf1d48b46a0\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xdf1d394a3a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-rx2sd.apps.aro.cilium-cluster.k38l.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"509ce29f121aa8889071be97ef0ea23faf908d0bfad41dc19a0232af98d72839","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9dwkh.apps.aro.x6z1d9t4s8t4n5n.sead.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"a602a5f022fbb4291aa237e264a072872cb19c74d32fbf75f8cb634f032e9cd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"dd7ef8065ee1042cfa5c3e7a73f09180f1300ed5a4e62e2f76b03643b5c75908","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-pfk2tm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c89b108af80\u003e: \n DNS for route host agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3c89b2105000\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-pfk2tm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3c89b108af80\u003e: \n DNS for route host agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3c89b2105000\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vbf5l.apps.aro.a2i2k8y4h2i2q2z.3d9a.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"c6a485e4e803566b4a409b38f15e0ed2ffb1819eb475ae010584f6e7ab74bf03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"68fc0f1ea6cf0f44536f6ef401fc1e1215329535859894d52835b6c6b59f5866","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2aeb303488e0\u003e: \n DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2aeb303488c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2aeb303488e0\u003e: \n DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2aeb303488c0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-g4jkv.apps.aro.s7l8k8t7p1f9x2x.w4l4.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13f4d3d4d6df90cd306313783d31231d8428475383f7424332fdcb9914b386a1","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-2lr49"}},{"row_id":"cc3e747d02c4984d7143d1607723bff8bb133b7394475d138d6f49ab25bec6ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113777030108549120","occurred_at":"2026-10-23T23:38:10Z","failed_at":"e2e","signature_id":"d9fbf23376585cdfaf80950f7a3f5e8696d3b0c22688297d50b6821493fe3062","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-q5x22b\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x21a3a7aea6e0\u003e: \n DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x21a3a7548980\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-q5x22b\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x21a3a7aea6e0\u003e: \n DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x21a3a7548980\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["13f4d3d4d6df90cd306313783d31231d8428475383f7424332fdcb9914b386a1","17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","3ce2ba89b8e9686083a83dadac7501168f4efe9a7eacff619946e50e9db1c762","509ce29f121aa8889071be97ef0ea23faf908d0bfad41dc19a0232af98d72839","7ec625aa187990188920942c47358d3eb1c7eda37bdf66a6d43c4b3578a43526","8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T23:00:02Z","started_at":"2026-10-23T23:00:02Z","completed_at":"2026-10-24T00:40:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:48:13.270426319Z","timing_metadata_checked_at":"2026-10-24T00:48:13.270426319Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T00:48:13.274671206Z","region_metadata_checked_at":"2026-10-24T00:48:13.274671206Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"2002f8b68de2da3c409a5079da3bb74839d87d1e28f519ea6e7c64f88e0daca4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"c3d8588a7330141641207b393ef0b056b3f565ec62f5715a57756d5035d5fa24","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x48191e04b0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x48191e04b0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"32af7b04567abc70e19cbd1985c153e9496be271e2ed5964fe0b17f3f569f69e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"f8ad7afc2b67ed6f03b36c92fec6e7ed4f59bd175316c494010190cf2fe6a1df","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2d5e4d930a10\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2d5e4d930a10\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"3bc9ec9d2b679b9b5d0243b2fdecc4de6dbdd1d579e6aa646773aa1c5578dfca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"489c87a4b234518c7dfe6b37a3d19c5c6af589cc404fb24f4b84bb4855abdbd3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-22-2jscs4\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0x1709fb679350\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x1709fb856180\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x1709fb856160\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x1709fb40e8c0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x1709fb7e92c0\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x1709fb7e92a0\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x1709fb8c7c20\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-22-2jscs4\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0x1709fb679350\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x1709fb856180\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x1709fb856160\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x1709fb40e8c0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x1709fb7e92c0\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x1709fb7e92a0\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x1709fb8c7c20\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"45e9751157451e8c6d0d0728e75ec9ace0ebf6b25263252932a5955d6ed70d2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"5124c660676e607323f38b7ee2b1cca5949648ceecce1866b6241aed16b10219","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-20-47zdm4\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2a6cf3f90c60\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x2a6cf3f90c40\u003e{\n observation: \u003c*errors.errorString | 0x2a6cf562de70\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x2a6cf3f90ba0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-20-47zdm4\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2a6cf3f90c60\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x2a6cf3f90c40\u003e{\n observation: \u003c*errors.errorString | 0x2a6cf562de70\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x2a6cf3f90ba0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"4d3d446666e38259592199e6eec12a2539342c767ff9f9aa00999d53c87cb84f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"f9b8e244e278e564970161a86e6652a63f58015bd2e7dc85de719e5bba6ef252","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-z69wtq\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2c961c50900\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*fmt.wrapError | 0x2c961c506a0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*fmt.wrapError | 0x2c961c505e0\u003e{\n msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*errors.errorString | 0x2c963abc0c0\u003e{\n s: \"rate: Wait(n=1) would exceed context deadline\",\n },\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-5-0-z69wtq\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2c961c50900\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*fmt.wrapError | 0x2c961c506a0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*fmt.wrapError | 0x2c961c505e0\u003e{\n msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n err: \u003c*errors.errorString | 0x2c963abc0c0\u003e{\n s: \"rate: Wait(n=1) would exceed context deadline\",\n },\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"7e72ea16056a3c8b2a7e915d1b0c31b24a48a2b53611c65122718bb06a9425af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"79dfce6cb10e96a30452837453f16449a11349eb353010336138903c85b23a84","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-v65qkr\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xbd03a06ce0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0xbd03a06ca0\u003e{\n observation: \u003c*errors.errorString | 0xbd05b0abd0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0xbd03a06c60\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-nightly-4-23-v65qkr\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xbd03a06ce0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0xbd03a06ca0\u003e{\n observation: \u003c*errors.errorString | 0xbd05b0abd0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0xbd03a06c60\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ebb8d128ac36a20abe9223f2c5925135d7f381d45a13394b4f2869aab4d48252","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\""}},{"row_id":"a0868c44c197d96526b17bc7b6b62e1c6526994bd27dca1889bb45a4567b761d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"a9d0787a49a9434b575f780199c840b15cab7a7f5c8097f81f3e40e8e6452e17","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0xcc47ebfda50\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0xcc47ebfda50\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"c7799bdff129328eaae903d0bbe7ce45e7871c6701f49c44473f421f280b6fe8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"e444cd22aa468d73f20a8d2ff0e73fd525cfd2b1f33d4eab9b629bfe695606fe","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x37eb02889a50\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x37eb02889a50\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"cabff14e928116b3f89fa7a92d0c8c1d648298f72aa0d5b0db509fe62bb0e558","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"bd72849f8bfb61f3950e07a620fb8b9bddc49d0f09058e4f6ff7f7eef02627f5","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2c66d40de440\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2c66d40de440\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"e7d598d4dbab2dd5237ff410b1abbaa2671f74f9d23ed010c7251dfcf056f94b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113767431099912192","occurred_at":"2026-10-23T23:00:02Z","failed_at":"e2e","signature_id":"e854c2baa0b762928bd1753c83c278e1be7a0f0d62fdeaf4af4e9edab0cc8f65","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-21-hwnzpg\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0x2be9a2dba918\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2be9a2235d00\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2be9a2235cc0\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2be9a29e6c80\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x2be9a21cc080\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2be9a21cc020\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2be9a3120140\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-21-hwnzpg\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0x2be9a2dba918\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2be9a2235d00\u003e{\n msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2be9a2235cc0\u003e{\n msg: \"failed to list services: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2be9a29e6c80\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n \u003c*fmt.wrapError | 0x2be9a21cc080\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n err: \u003c*fmt.wrapError | 0x2be9a21cc020\u003e{\n msg: \"failed to list all APIServices: Unauthorized\",\n err: \u003c*errors.StatusError | 0x2be9a3120140\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Unauthorized\",\n Reason: \"Unauthorized\",\n Details: nil,\n Code: 401,\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","ebb8d128ac36a20abe9223f2c5925135d7f381d45a13394b4f2869aab4d48252","fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113756442623217664","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T22:16:22Z","started_at":"2026-10-23T22:16:22Z","completed_at":"2026-10-24T00:42:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T00:48:13.112044066Z","timing_metadata_checked_at":"2026-10-24T00:48:13.112044066Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T00:48:13.115128861Z","region_metadata_checked_at":"2026-10-24T00:48:13.115128861Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113725294606028800","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T20:12:29Z","started_at":"2026-10-23T20:12:29Z","completed_at":"2026-10-23T22:36:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T22:52:53.572432402Z","timing_metadata_checked_at":"2026-10-23T22:52:53.572432402Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T22:52:53.575279792Z","region_metadata_checked_at":"2026-10-23T22:52:53.575279792Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T20:07:29Z","started_at":"2026-10-23T20:07:29Z","completed_at":"2026-10-23T22:28:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T22:32:43.515087465Z","timing_metadata_checked_at":"2026-10-23T22:32:43.515087465Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T22:32:43.517828957Z","region_metadata_checked_at":"2026-10-23T22:32:43.517828957Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"05e405ef0a91755382a8e14a6646ccef02f5b7a15a5a24fb75a3389d583ad4c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"16e5dc89de6f908080887b19103bb8d46e6a6d30176df898a38a625fe63e5380","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:526]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x33122d4ce1c0\u003e: \n DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x33122d4ce1a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:526]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x33122d4ce1c0\u003e: \n DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x33122d4ce1a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-2lr49.apps.aro.ingress-cert-le.xc5o.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13f4d3d4d6df90cd306313783d31231d8428475383f7424332fdcb9914b386a1","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-2lr49"}},{"row_id":"1ee096658e91a12402f791bcdba4dda831fdb8423f2420a39efdcf0041f1a29a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"cdfd5eadc24e0be0c150b2244b8232f0a89d3c7047aecd3074d3a83bba715ea6","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x398579d77aa0\u003e: \n DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x398579d77a60\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:169]: sample application must serve the delivered default certificate\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x398579d77aa0\u003e: \n DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x398579d77a60\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lpqxs.apps.aro.d1o5s8c9e5q6t1e.xrjv.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"169f70dba2351443907226ac5ccc1485fb8e510b011ab14b2d8af60b14150235","failure_pattern":"DNS for route host agnhost-e2e-sample-app-lpqxs","search_query":"DNS for route host agnhost-e2e-sample-app-lpqxs"}},{"row_id":"226d92a1e01fe246554079aec9dfc6749b784f07f6dba1d454bc0fa04bffb48f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"8e5db3d65c7b7608ee7e78aa4ee79fca7ba6f2e43840a7bfc37e8268baf454dc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-b48sqx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e8c038e83a0\u003e: \n DNS for route host agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3e8c03dd0340\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-b48sqx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3e8c038e83a0\u003e: \n DNS for route host agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x3e8c03dd0340\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dgfpk.apps.aro.c2c1j7v6o7x9e3w.no9b.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"3c33d85f47ea2fa7b3a0a30833dd7567444bc5aa74859a2c4e26328ee037836c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"476aa5c24e557ad6a577878e82a38cad972775741e8c95bde27dcf8780a1adf4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-hb5rp9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x38ad847b15a0\u003e: \n DNS for route host agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x38ad8403b0a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-hb5rp9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x38ad847b15a0\u003e: \n DNS for route host agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x38ad8403b0a0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-nplh9.apps.aro.p4i4z6i9y0h8m3b.r4vt.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"4959994d276e8f75f1e1ee4c5ddf0967a55cb0b2f8ca3b59f43fc7a91647df6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"1bd3c4c4ad3a9bdeb00d2c5694e282e664d811eb31be8ef577772f8ad9b54e05","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-9d6fpw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x21815df3d960\u003e: \n DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x21815eb44080\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-9d6fpw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x21815df3d960\u003e: \n DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x21815eb44080\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"8830ddcb10421af354c9ebf06154d103c327f03440ef615859fcdd566a34a83a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"bda22742bb4bcfe82726d83de21fb4e400993a95b279453234202a917cefb495","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0x397e13438ac8\u003e: \n context deadline exceeded\n cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*errors.errorString | 0x397e134eabf0\u003e{\n s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0x397e13438ac8\u003e: \n context deadline exceeded\n cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n {\n errs: [\n \u003ccontext.deadlineExceededError\u003e{},\n \u003c*errors.errorString | 0x397e134eabf0\u003e{\n s: \"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\",\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f26d2f1140adc9b13b39b836a7ca29a860f5c4cb3811309f672f97a11466f4d","failure_pattern":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True","search_query":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)"}},{"row_id":"89de65041402cf4bbee10fc0342d7c810aef36cc57f9499b104cfb3e5c39da35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"3bfc2a89887bfd392938b2d517fdbf090bf267eaca9f9ccad980b9907856ef39","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 634.146s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n \u003c[]string | len:2, cap:2\u003e: [\n \"console (Available=False)\",\n \"ingress (Available=False)\",\n ]\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:378]: Timed out after 634.146s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:377 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n \u003c[]string | len:2, cap:2\u003e: [\n \"console (Available=False)\",\n \"ingress (Available=False)\",\n ]\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b286b88e4cdf2beb25fd0ed17c671477745b97698c63efca626d18c2b4af0e4","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]"}},{"row_id":"91c74b2e1f274df6756a26527703eb106a10f49c1a345ca5f7e8febfb0a58b40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"aaeb63305e9f896148e97071cab84607802b6d3dc40093f2e77fe61a4284ba60","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-mx6tbf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x10452515d3c0\u003e: \n DNS for route host agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1045256005e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-mx6tbf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x10452515d3c0\u003e: \n DNS for route host agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1045256005e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-p2dgw.apps.aro.g8l5h2e8g9b9a4u.ex01.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-67lsh.apps.aro.i0t9o8c4q5f8r2m.4m1k.eastus2.aroapp-hcp.io on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"cc196cc014a82dcccf1b4953d4a159d33cbbe18435758d978d90ee80c74ec7b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113724010066874368","occurred_at":"2026-10-23T20:07:29Z","failed_at":"e2e","signature_id":"baa9afa3bcbcd0c44804b70a02b3154a77e55643457f2cbdcc513051780298c8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-fvww2g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x194223a493c0\u003e: \n DNS for route host agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194223dd6660\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster \"cluster-candidate-4-23-fvww2g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x194223a493c0\u003e: \n DNS for route host agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194223dd6660\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-wzr8t.apps.aro.t9b6r2z8e5x9w7g.5hae.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve: DNS for agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-27qvf.apps.aro.l8d0x7x9p0e3b5l.tq9n.eastus2.aroapp-hcp.io on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["13f4d3d4d6df90cd306313783d31231d8428475383f7424332fdcb9914b386a1","169f70dba2351443907226ac5ccc1485fb8e510b011ab14b2d8af60b14150235","17960e7399b10596ca3db9acfee2784f8057d16e1a50930e2429e794d4813ec1","4b286b88e4cdf2beb25fd0ed17c671477745b97698c63efca626d18c2b4af0e4","6f26d2f1140adc9b13b39b836a7ca29a860f5c4cb3811309f672f97a11466f4d","8549e492eb914214ed17a212525eac80f1d245385bd55db4bd0b3caaa6c9e378"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113679461063856128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T17:10:28Z","started_at":"2026-10-23T17:10:28Z","completed_at":"2026-10-23T19:13:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T19:23:43.847126443Z","timing_metadata_checked_at":"2026-10-23T19:23:43.847126443Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T19:23:43.849951936Z","region_metadata_checked_at":"2026-10-23T19:23:43.849951936Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113640155058278400","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T14:34:17Z","started_at":"2026-10-23T14:34:17Z","completed_at":"2026-10-23T16:39:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T16:54:37.745347123Z","timing_metadata_checked_at":"2026-10-23T16:54:37.745347123Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T16:54:37.750662606Z","region_metadata_checked_at":"2026-10-23T16:54:37.750662606Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"50707f02934eaf9294c9a786c81fb6d5ec8573934a401c50bd410fd7477f0731","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113640155058278400","occurred_at":"2026-10-23T14:34:17Z","failed_at":"e2e","signature_id":"83894b496f209ee26cbf0e4d1f1a178c6a6d6f9bdf0c14187a9b22ad01f74877","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x9a8a07812a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-npbbgrcnjvjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n \"name\": \"5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T14:54:47.282384682Z\",\n \"endTime\": \"2026-10-23T15:13:53.224517483Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n correlation ID: 90989f90-f502-41d8-955e-974a04d1e9d9\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x9a8a07812a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-npbbgrcnjvjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n \"name\": \"5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T14:54:47.282384682Z\",\n \"endTime\": \"2026-10-23T15:13:53.224517483Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-2vwr5 to exist\\nMachine d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-npbbgrcnjvjf--managed/providers/Microsoft.Compute/virtualMachines/d2h8j1q0z4a1f4u-np-1-ntqn4-rkqlm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/5c43f0b5-5e0d-4c5c-89a9-5d1f04167fe7\n correlation ID: 90989f90-f502-41d8-955e-974a04d1e9d9\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d573316c54ef43f3ac68e25b1560c7897db987a56165add667abc693301db0a0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d573316c54ef43f3ac68e25b1560c7897db987a56165add667abc693301db0a0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113595013265887232","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T11:34:54Z","started_at":"2026-10-23T11:34:54Z","completed_at":"2026-10-23T13:50:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:05:35.027153259Z","timing_metadata_checked_at":"2026-10-23T14:05:35.027153259Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:05:35.02976455Z","region_metadata_checked_at":"2026-10-23T14:05:35.02976455Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113594510964428800","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T11:32:50Z","started_at":"2026-10-23T11:32:50Z","completed_at":"2026-10-23T13:49:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:05:34.895074414Z","timing_metadata_checked_at":"2026-10-23T14:05:34.895074414Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:05:34.898205903Z","region_metadata_checked_at":"2026-10-23T14:05:34.898205903Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113594207997267968","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T11:31:42Z","started_at":"2026-10-23T11:31:42Z","completed_at":"2026-10-23T13:48:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:05:34.79178367Z","timing_metadata_checked_at":"2026-10-23T14:05:34.79178367Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:05:34.79485436Z","region_metadata_checked_at":"2026-10-23T14:05:34.79485436Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f7de682c8cf27a084636bb48ef0608e5cd9d675428780a917da88c6999f829a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113594207997267968","occurred_at":"2026-10-23T11:31:42Z","failed_at":"e2e","signature_id":"a4e598ab2e945986cd1ad69b1bb3c65fbdf94c05e97f34879cec8e6778f23b99","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ef724ca1cc0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gnnvxwc94fcv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n \"name\": \"eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:03:20.324115794Z\",\n \"endTime\": \"2026-10-23T13:22:20.156214194Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n correlation ID: 5b15bb34-6e8a-47a7-ac3b-f4e7ecb8ef82\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x2ef724ca1cc0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-gnnvxwc94fcv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n \"name\": \"eb80abc6-2e4c-4f05-8321-ab95d4028e3a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:03:20.324115794Z\",\n \"endTime\": \"2026-10-23T13:22:20.156214194Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-cv8p2 to exist\\nMachine oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gnnvxwc94fcv--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-5g7jb-qgh76 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/switzerlandnorth/hcpOperationStatuses/eb80abc6-2e4c-4f05-8321-ab95d4028e3a\n correlation ID: 5b15bb34-6e8a-47a7-ac3b-f4e7ecb8ef82\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6fe02653aee74c57d72dfff05e4f572539da57d10c19d6d05ba4d0781d80c774","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6fe02653aee74c57d72dfff05e4f572539da57d10c19d6d05ba4d0781d80c774"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113593585990373376","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T11:29:14Z","started_at":"2026-10-23T11:29:14Z","completed_at":"2026-10-23T13:46:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T14:05:34.661583719Z","timing_metadata_checked_at":"2026-10-23T14:05:34.661583719Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T14:05:34.665585105Z","region_metadata_checked_at":"2026-10-23T14:05:34.665585105Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113545423367966720","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-23T08:17:48Z","started_at":"2026-10-23T08:17:48Z","completed_at":"2026-10-23T10:34:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T10:38:32.012987533Z","timing_metadata_checked_at":"2026-10-23T10:38:32.012987533Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T10:38:32.016254423Z","region_metadata_checked_at":"2026-10-23T10:38:32.016254423Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T02:00:29Z","started_at":"2026-10-23T02:00:29Z","completed_at":"2026-10-23T04:07:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T04:25:07.464561497Z","timing_metadata_checked_at":"2026-10-23T04:25:07.464561497Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-23T04:25:07.468237285Z","region_metadata_checked_at":"2026-10-23T04:25:07.468237285Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"208ea226c1a440e28cecf9ed4a1c4c4a1c521384072bd09feb0efc0c45d4fda6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"951e3efbb76f026e0f9c0427659ee8ef966b4d7a1594708f3b87a3b80dade019","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2b0e7254f60\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2b0e7254f60\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"6d078e209ed99e5a1932f77d64eed5a86db779098b6b5aa69632e4f254c380bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"4a987599543343a0a61051617564a6330ab4d61da2e4fe37b0c8507e51dec2dc","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc1401787740\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc1401787740\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"88ab44f9b8308647188a8af198f47781844eb55d7ff0a766f09dfcd606af9c0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"2d6ae229654176a356f5c870d83add62836595109dbb7a96870fa8c8c6a4e33b","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x24769a10c7f0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x24769a10c7f0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"936acb391d0e3b7c3eb313daae91c2accb8b0e6fd4caa5ab03eca336c9ee9d55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"8b45e8c93fa73392b7a5435ed12dc6c9c04d541483629a0f2c963d83febedd8d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-20-8zbqmf\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x15ca294921a0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x15ca29492180\u003e{\n observation: \u003c*errors.errorString | 0x15ca28eb6740\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x15ca294920a0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-20-8zbqmf\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x15ca294921a0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x15ca29492180\u003e{\n observation: \u003c*errors.errorString | 0x15ca28eb6740\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x15ca294920a0\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"a1ea09efc5f1e3b7cbd32851301ed4e0386ba8e7a1800490d8177a89aeb15308","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"8adea4c35fe3f9117bdf109bb92ed50cc3312cf016d9f42e8bb8d7d02a60e613","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-22-dppnl7\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x254a74952740\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x254a74952660\u003e{\n observation: \u003c*errors.errorString | 0x254a747beb00\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x254a74952640\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-22-dppnl7\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x254a74952740\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x254a74952660\u003e{\n observation: \u003c*errors.errorString | 0x254a747beb00\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x254a74952640\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"b53b9028977a47cd7b289f5347bb9c557b56e76dafe228ccf3db970b86f682b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"8ae7224beef582a60fc95922521582cea88650e7efa805e02a16af12f707e016","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1ee734123b50\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1ee734123b50\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"d782d8bfba7ddd178468292b2d87e63a41e97fba0e524a581f7ce09b1f47c838","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"5dc7704bcbce0f4a7f9f81907bd6a202126d16c8173b589ea3e9a7efd501998e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-drlc89\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3bc58413f6c0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x3bc584326750\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-drlc89\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3bc58413f6c0\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x3bc584326750\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"f81e0cbe148079ae65e9efc902cf18740f223367f88e396b32d886e2bd82527c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"13d3f7c24d1951f7eccaeb591f09a7ad27b72a1f0d3c63a0a590154ec1d02b2d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-21-b4mmvn\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x123062b14220\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x123062b141a0\u003e{\n observation: \u003c*errors.errorString | 0x1230626cb5f0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x123062b14140\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-21-b4mmvn\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x123062b14220\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*verifiers.pollObservationError | 0x123062b141a0\u003e{\n observation: \u003c*errors.errorString | 0x1230626cb5f0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n interruption: \u003c*fmt.wrapError | 0x123062b14140\u003e{\n msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"f9d75db949ee8274d022d90a899360ddaf211f0f331d3cdd7f570381853fc533","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"083741f16af6bde01a15c32f163b36e9e075c1b8a96f107ca16e1a1830aa7995","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1b6937cc7c90\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1b6937cc7c90\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"fb5d358028c4d66266e7122d85ccc26e9797058b41441a3f63ec252bfda58ea9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113450457643880448","occurred_at":"2026-10-23T02:00:29Z","failed_at":"e2e","signature_id":"59e0905d51abc3f260e60eaccaab70b634685836f78308d19447082523bd7935","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-gqg8kx\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3c86c376e000\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x3c86c4fb96a0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:216]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-gqg8kx\"\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3c86c376e000\u003e: \n VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.003s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x3c86c4fb96a0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113450198238760960","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-23T01:59:27Z","started_at":"2026-10-23T01:59:27Z","completed_at":"2026-10-23T02:29:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-23T02:39:39.891283989Z","timing_metadata_checked_at":"2026-10-23T02:39:39.891283989Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-23T02:39:39.89473397Z","region_metadata_checked_at":"2026-10-23T03:42:50.069210121Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"66d27573a16792f43140363024af37ae6bbdcc35b774b943f1cf0a7bac631529","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113450198238760960","occurred_at":"2026-10-23T01:59:27Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}}]}]}