{"meta":{"start_date":"2026-06-20","end_date":"2026-06-20","start_at":"2026-06-20T00:00:00Z","end_at":"2026-06-21T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T08:27:01Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":13,"failed_runs":10,"raw_occurrences":34,"matched_occurrences":33,"runs_affected":9},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":8,"runs_affected":6,"failed_runs":6,"run_impact_percent":46.15384615384615,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,0,8],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-01","2026-06-08"],"prior_runs_affected":56,"prior_last_seen_at":"2026-06-14T23:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-55jt6c\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00146eb10\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-55jt6c, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-55jt6c in resource group rg-zstream-upgrade-4-22-gmxf8f, error: failed waiting for cluster=\"cluster-zstream-4-22-55jt6c\" in resourcegroup=\"rg-zstream-upgrade-4-22-gmxf8f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-55jt6c in resource group rg-zstream-upgrade-4-22-gmxf8f, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-55jt6c\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00146eb10\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-55jt6c, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-55jt6c in resource group rg-zstream-upgrade-4-22-gmxf8f, error: failed waiting for cluster=\"cluster-zstream-4-22-55jt6c\" in resourcegroup=\"rg-zstream-upgrade-4-22-gmxf8f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-55jt6c in resource group rg-zstream-upgrade-4-22-gmxf8f, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-lvjsdl/cluster-nightly-5-0-hxzbgf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016204e0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-hxzbgf, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-hxzbgf in resource group rg-nightly-5-0-lvjsdl, error: failed waiting for cluster=\"cluster-nightly-5-0-hxzbgf\" in resourcegroup=\"rg-nightly-5-0-lvjsdl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-hxzbgf in resource group rg-nightly-5-0-lvjsdl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-lvjsdl/cluster-nightly-5-0-hxzbgf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016204e0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-hxzbgf, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-hxzbgf in resource group rg-nightly-5-0-lvjsdl, error: failed waiting for cluster=\"cluster-nightly-5-0-hxzbgf\" in resourcegroup=\"rg-nightly-5-0-lvjsdl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-hxzbgf in resource group rg-nightly-5-0-lvjsdl, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-lklm69/cluster-candidate-4-22-f4gmfq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014d4690\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-lklm69/cluster-candidate-4-22-f4gmfq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014d4690\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2068469052392607744","occurred_at":"2026-06-20T23:00:27Z","signature_id":"1374440a9b7fdf0dd4c5846fb1575b98eb5b806accbce15929c0e84a45cc26e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2068469052392607744","occurred_at":"2026-06-20T23:00:27Z","signature_id":"8d531303f114240ae47837d3f93a8dd60e92225d821d454820c92e5a5ba27fc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"3f80849ae13bf039b50b9f059c7ba4fb1f749b1f47f6ac1f11ee3c7ab6e3815b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068443878926585856","occurred_at":"2026-06-20T21:20:25Z","signature_id":"17d655e6d8990c66981568b9fe20a63447579ff5bad898d4b6ec61e35a8a9888","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068443878926585856","occurred_at":"2026-06-20T21:20:25Z","signature_id":"5a0423b3923dc89a368332a2790ff39707801c91b57f2adb24e38d2edda0fd61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"0275320401f1ba8581c52de46c654704d3f59df8b34a9f21dacf93ee711e9816","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068216057461477376","occurred_at":"2026-06-20T06:15:08Z","signature_id":"1e2e5dcf69c1b15b0879893ecc53787a77e157be5eb9f8f914bdb5e424da68aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068215937068175360","occurred_at":"2026-06-20T06:14:39Z","signature_id":"3a75c96a10ecf612dd2f7d814a539cc03f1bbd30ad0eca88d410a8c39cb508a3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":15.384615384615385,"anchor_occurrences":7,"after_last_push_count":17,"trend_counts":[0,0,0,0,0,10,7],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to verify simple web app on back-level cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f540e0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-zgg49.apps.aro.h8q7w4u9x7c7g6v.x1ie.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-zgg49.apps.aro.h8q7w4u9x7c7g6v.x1ie.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to verify simple web app on back-level cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f540e0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-zgg49.apps.aro.h8q7w4u9x7c7g6v.x1ie.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-zgg49.apps.aro.h8q7w4u9x7c7g6v.x1ie.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-wxppcl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00012ae20\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-4d2pm.apps.aro.f2j5d7e4q4x3n6o.g26p.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-4d2pm.apps.aro.f2j5d7e4q4x3n6o.g26p.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-wxppcl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00012ae20\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-4d2pm.apps.aro.f2j5d7e4q4x3n6o.g26p.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-4d2pm.apps.aro.f2j5d7e4q4x3n6o.g26p.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-6z4flh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000954aa0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-dg7r4.apps.aro.k4f9f7h5q5s9h1o.o4cb.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-dg7r4.apps.aro.k4f9f7h5q5s9h1o.o4cb.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-6z4flh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000954aa0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-dg7r4.apps.aro.k4f9f7h5q5s9h1o.o4cb.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-dg7r4.apps.aro.k4f9f7h5q5s9h1o.o4cb.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"010b43b1e276924b59a68771f3466bc2f1fc8a5a375e7d1e7657e0f0c867c176","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"5ae9f4af0de562607c311fca891af1f286068eb5c6634c83d40b2860e29f8a3f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"a04fb3f7f155acd62abe2cf3dddc4d784297767710d690469fb9a348af3ab4a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"f34d6af338b8454dc2204b005de839554fce4302748615240ec6a748683432ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"2c746df90ce47ce86f765b37e298a40847ec8905f8537e45167732ed3aa8f5f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"4c0bb84b47552cb6c935e3c0563b31ab7117a9c9fbe4151088f52951f0cdf408","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"7e708cce8624ecb4df2576f7d6e3253a9bfce775ae2d53fe9470231ef30ada56","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"78754e29456c1fc88cbae908323785a403c4346735c925fc3102d3c04ce7e5df","failure_pattern":"VerifySimpleWebApp failed: route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":15.384615384615385,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,2,4],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0021b6000\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-8872n.apps.aro.cilium-cluster.moy3.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-8872n.apps.aro.cilium-cluster.moy3.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0021b6000\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-8872n.apps.aro.cilium-cluster.moy3.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-8872n.apps.aro.cilium-cluster.moy3.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012542a0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-j5bdd.apps.aro.cilium-cl.17y9.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-j5bdd.apps.aro.cilium-cl.17y9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012542a0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-j5bdd.apps.aro.cilium-cl.17y9.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-j5bdd.apps.aro.cilium-cl.17y9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011fe378\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011fe378\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"19a8c1da69ee46dc328267e29bc50e0a088d6a1ecdc3d076590d7368d6a3a662","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"495f7bdf204bbd214858933af31f5d044a5bcb1b8b501916ba253c491daaf0a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"1111515492f30e2408acccb3968c1c03c551b248e3cde5b1657318baf89edc91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"cc133a77418a5acbe06aa2926952a966fa6b5a5c0d00dc0d96988b71409fe672","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":15.384615384615385,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["go:2125 +0x47 fp=0xc000be4d28 sp=0xc000be4cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc00162c1e0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc000be4d60 sp=0xc000be4d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc00162c1e0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc000be4d80 sp=0xc000be4d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc001359d40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc000be4fc8 sp=0xc000be4d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc000be4fe0 sp=0xc000be4fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000be4fe8 sp=0xc000be4fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 173\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 189 gp=0xc001001dc0 m=nil [select, 1 minutes]:\nruntime.gopark(0xc0016b3f48?, 0x2?, 0xc0?, 0x3d?, 0xc0016b3ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0016b3d78 sp=0xc0016b3d58 pc=0x2f1e50e\nruntime.selectgo(0xc0016b3f48, 0xc0016b3ef0, 0xc000c97240?, 0x0, 0xc0018e37d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0016b3eb8 sp=0xc0016b3d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc001359d40)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc0016b3fc8 sp=0xc0016b3eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc0016b3fe0 sp=0xc0016b3fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0016b3fe8 sp=0xc0016b3fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 173\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 235 gp=0xc0016a4fc0 m=nil [select]:\nruntime.gopark(0xc0013df768?, 0x2?, 0x90?, 0x1f?, 0xc0013df75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0013df5e8 sp=0xc0013df5c8 pc=0x2f1e50e\nruntime.selectgo(0xc0013df768, 0xc0013df758, 0xc000f5d760?, 0x0, 0x36ecd5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0013df728 sp=0xc0013df5e8 pc=0x2ef92f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x92b8658, 0xc0018e0b60}, 0x8704ddd?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc0013df798 sp=0xc0013df728 pc=0x36ec4b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x930a440, {0x92b8658, 0xc0018e0b60}, 0x2f1baf3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc0013dfa98 sp=0xc0013df798 pc=0x3ded372\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x92b8658, 0xc0018e0b60}, 0xc001037308, {0xc0010125e8, 0x12}, {0xc001546108, 0x12})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc0013dfc20 sp=0xc0013dfa98 pc=0x6c2ef65\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x92b8658?, 0xc0018e0b60?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc0013dfcc8 sp=0xc0013dfc20 pc=0x6c2e992\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x92b8658?, 0xc0018e0b60?}, 0xc0018e0b70?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc0013dfd20 sp=0xc0013dfcc8 pc=0x369fa82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x92b8658, 0xc0018e0b60}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc0013dfe68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc0013dfd98 sp=0xc0013dfd20 pc=0x369db2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x92b85e8, 0xc0008bb540}, 0xc001037308, {0xc0010125e8, 0x12}, {0xc001546108, 0x12}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc0013dfef0 sp=0xc0013dfd98 pc=0x6c2e705\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc0013dff78 sp=0xc0013dfef0 pc=0x6c325fe\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc0013dffe0 sp=0xc0013dff78 pc=0x4a4be30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0013dffe8 sp=0xc0013dffe0 pc=0x2f26cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 214\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x2f28ae3\nrdx    0x0\nrdi    0xc90cc70\nrsi    0x80\nrbp    0x7ffcdfaaa7b0\nrsp    0x7ffcdfaaa768\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x1c\nr13    0x1\nr14    0xc90a6e0\nr15    0x7\nrip    0x2f28ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\ngo:2125 +0x47 fp=0xc000be4d28 sp=0xc000be4cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc00162c1e0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc000be4d60 sp=0xc000be4d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc00162c1e0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc000be4d80 sp=0xc000be4d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc001359d40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc000be4fc8 sp=0xc000be4d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc000be4fe0 sp=0xc000be4fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000be4fe8 sp=0xc000be4fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 173\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 189 gp=0xc001001dc0 m=nil [select, 1 minutes]:\nruntime.gopark(0xc0016b3f48?, 0x2?, 0xc0?, 0x3d?, 0xc0016b3ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0016b3d78 sp=0xc0016b3d58 pc=0x2f1e50e\nruntime.selectgo(0xc0016b3f48, 0xc0016b3ef0, 0xc000c97240?, 0x0, 0xc0018e37d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0016b3eb8 sp=0xc0016b3d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc001359d40)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc0016b3fc8 sp=0xc0016b3eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc0016b3fe0 sp=0xc0016b3fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0016b3fe8 sp=0xc0016b3fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 173\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 235 gp=0xc0016a4fc0 m=nil [select]:\nruntime.gopark(0xc0013df768?, 0x2?, 0x90?, 0x1f?, 0xc0013df75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0013df5e8 sp=0xc0013df5c8 pc=0x2f1e50e\nruntime.selectgo(0xc0013df768, 0xc0013df758, 0xc000f5d760?, 0x0, 0x36ecd5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0013df728 sp=0xc0013df5e8 pc=0x2ef92f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x92b8658, 0xc0018e0b60}, 0x8704ddd?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc0013df798 sp=0xc0013df728 pc=0x36ec4b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x930a440, {0x92b8658, 0xc0018e0b60}, 0x2f1baf3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc0013dfa98 sp=0xc0013df798 pc=0x3ded372\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x92b8658, 0xc0018e0b60}, 0xc001037308, {0xc0010125e8, 0x12}, {0xc001546108, 0x12})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc0013dfc20 sp=0xc0013dfa98 pc=0x6c2ef65\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x92b8658?, 0xc0018e0b60?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc0013dfcc8 sp=0xc0013dfc20 pc=0x6c2e992\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x92b8658?, 0xc0018e0b60?}, 0xc0018e0b70?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc0013dfd20 sp=0xc0013dfcc8 pc=0x369fa82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x92b8658, 0xc0018e0b60}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc0013dfe68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc0013dfd98 sp=0xc0013dfd20 pc=0x369db2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x92b85e8, 0xc0008bb540}, 0xc001037308, {0xc0010125e8, 0x12}, {0xc001546108, 0x12}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc0013dfef0 sp=0xc0013dfd98 pc=0x6c2e705\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc0013dff78 sp=0xc0013dfef0 pc=0x6c325fe\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc0013dffe0 sp=0xc0013dff78 pc=0x4a4be30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0013dffe8 sp=0xc0013dffe0 pc=0x2f26cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 214\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x2f28ae3\nrdx    0x0\nrdi    0xc90cc70\nrsi    0x80\nrbp    0x7ffcdfaaa7b0\nrsp    0x7ffcdfaaa768\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x1c\nr13    0x1\nr14    0xc90a6e0\nr15    0x7\nrip    0x2f28ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","53 fp=0xc0000c9d80 sp=0xc0000c9d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0010a6b40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0000c9fc8 sp=0xc0000c9d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0000c9fe0 sp=0xc0000c9fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c9fe8 sp=0xc0000c9fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 212\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 236 gp=0xc000e1ec40 m=nil [select, 2 minutes]:\nruntime.gopark(0xc001303f48?, 0x2?, 0xc0?, 0x3d?, 0xc001303ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000382d78 sp=0xc000382d58 pc=0x2f1e50e\nruntime.selectgo(0xc000382f48, 0xc001303ef0, 0xc0008e8b40?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000382eb8 sp=0xc000382d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc0018445a0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000382fc8 sp=0xc000382eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000382fe0 sp=0xc000382fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000382fe8 sp=0xc000382fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 197\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 235 gp=0xc000682e00 m=nil [IO wait, 2 minutes]:\nruntime.gopark(0xc0000d5760?, 0xc001105788?, 0xc5?, 0xbe?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001105768 sp=0xc001105748 pc=0x2f1e50e\nruntime.netpollblock(0x2f88ad8?, 0x2eaa066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0011057a0 sp=0xc001105768 pc=0x2eddfb7\ninternal/poll.runtime_pollWait(0x7f51f5ba3e00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0011057c0 sp=0xc0011057a0 pc=0x2f1cda5\ninternal/poll.(*pollDesc).wait(0xc001866680?, 0xc001728a00?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0011057e8 sp=0xc0011057c0 pc=0x2fab907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc001866680, {0xc001728a00, 0x2500, 0x2500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc001105880 sp=0xc0011057e8 pc=0x2facd39\nnet.(*netFD).Read(0xc001866680, {0xc001728a00?, 0xc001728a00?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0011058c8 sp=0xc001105880 pc=0x31712e5\nnet.(*conn).Read(0xc000d6aa10, {0xc001728a00?, 0x7f51f42af700?, 0x7f523c789f30?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc001105910 sp=0xc0011058c8 pc=0x3183385\ncrypto/tls.(*atLeastReader).Read(0xc001800f60, {0xc001728a00?, 0x240a?, 0x0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc001105958 sp=0xc001105910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000c129a8, {0x9220320, 0xc001800f60})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0011059b0 sp=0xc001105958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000c12708, {0x9221e00, 0xc000d6aa10}, 0xc001105a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0011059e8 sp=0xc0011059b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c12708, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc001105c60 sp=0xc0011059e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c12708, {0xc001892000, 0x1000, 0xc0008673b0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc001105cc8 sp=0xc001105c60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc0018445a0, {0xc001892000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc001105d28 sp=0xc001105cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc000e9db60)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc001105d60 sp=0xc001105d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc000e9db60, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc001105d80 sp=0xc001105d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0018445a0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc001105fc8 sp=0xc001105d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc001105fe0 sp=0xc001105fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001105fe8 sp=0xc001105fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 197\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax    0xca\nrbx    0x0\nrcx    0x2f28ae3\nrdx    0x0\nrdi    0xc90cc70\nrsi    0x80\nrbp    0x7fff4aa32180\nrsp    0x7fff4aa32138\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x7fff4aa32180\nr13    0x1\nr14    0xc90a6e0\nr15    0x5\nrip    0x2f28ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n53 fp=0xc0000c9d80 sp=0xc0000c9d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0010a6b40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0000c9fc8 sp=0xc0000c9d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0000c9fe0 sp=0xc0000c9fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c9fe8 sp=0xc0000c9fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 212\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 236 gp=0xc000e1ec40 m=nil [select, 2 minutes]:\nruntime.gopark(0xc001303f48?, 0x2?, 0xc0?, 0x3d?, 0xc001303ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000382d78 sp=0xc000382d58 pc=0x2f1e50e\nruntime.selectgo(0xc000382f48, 0xc001303ef0, 0xc0008e8b40?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000382eb8 sp=0xc000382d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc0018445a0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000382fc8 sp=0xc000382eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000382fe0 sp=0xc000382fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000382fe8 sp=0xc000382fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 197\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 235 gp=0xc000682e00 m=nil [IO wait, 2 minutes]:\nruntime.gopark(0xc0000d5760?, 0xc001105788?, 0xc5?, 0xbe?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001105768 sp=0xc001105748 pc=0x2f1e50e\nruntime.netpollblock(0x2f88ad8?, 0x2eaa066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0011057a0 sp=0xc001105768 pc=0x2eddfb7\ninternal/poll.runtime_pollWait(0x7f51f5ba3e00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0011057c0 sp=0xc0011057a0 pc=0x2f1cda5\ninternal/poll.(*pollDesc).wait(0xc001866680?, 0xc001728a00?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0011057e8 sp=0xc0011057c0 pc=0x2fab907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc001866680, {0xc001728a00, 0x2500, 0x2500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc001105880 sp=0xc0011057e8 pc=0x2facd39\nnet.(*netFD).Read(0xc001866680, {0xc001728a00?, 0xc001728a00?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0011058c8 sp=0xc001105880 pc=0x31712e5\nnet.(*conn).Read(0xc000d6aa10, {0xc001728a00?, 0x7f51f42af700?, 0x7f523c789f30?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc001105910 sp=0xc0011058c8 pc=0x3183385\ncrypto/tls.(*atLeastReader).Read(0xc001800f60, {0xc001728a00?, 0x240a?, 0x0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc001105958 sp=0xc001105910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000c129a8, {0x9220320, 0xc001800f60})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0011059b0 sp=0xc001105958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000c12708, {0x9221e00, 0xc000d6aa10}, 0xc001105a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0011059e8 sp=0xc0011059b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c12708, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc001105c60 sp=0xc0011059e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c12708, {0xc001892000, 0x1000, 0xc0008673b0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc001105cc8 sp=0xc001105c60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc0018445a0, {0xc001892000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc001105d28 sp=0xc001105cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc000e9db60)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc001105d60 sp=0xc001105d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc000e9db60, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc001105d80 sp=0xc001105d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0018445a0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc001105fc8 sp=0xc001105d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc001105fe0 sp=0xc001105fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001105fe8 sp=0xc001105fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 197\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax    0xca\nrbx    0x0\nrcx    0x2f28ae3\nrdx    0x0\nrdi    0xc90cc70\nrsi    0x80\nrbp    0x7fff4aa32180\nrsp    0x7fff4aa32138\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x7fff4aa32180\nr13    0x1\nr14    0xc90a6e0\nr15    0x5\nrip    0x2f28ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"ea6368dc87e56ff73bb14bc32d785a4af01fd962c88192375df0b5e51b063f4a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"12615c31a81b23d87c528a95f1eee5e33503c3eea5de7e32d04e9c7147e5f757","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":15.384615384615385,"anchor_occurrences":2,"after_last_push_count":25,"trend_counts":[0,0,0,1,0,3,2],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-08"],"prior_runs_affected":21,"prior_last_seen_at":"2026-06-12T19:09:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068432057893130240","occurred_at":"2026-06-20T20:33:26Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":15.384615384615385,"anchor_occurrences":2,"after_last_push_count":9,"trend_counts":[0,1,2,0,1,1,2],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-18","2026-06-08"],"prior_runs_affected":11,"prior_last_seen_at":"2026-06-13T23:00:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc00160ea98\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc00160ea98\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00106a780\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-26pckg, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26pckg\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00106a780\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-26pckg, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26pckg\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"61588617151eb55006027d519d5dbf47bf1b0f513822a4897fbfdb0dac1b1198","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068399947908124672","occurred_at":"2026-06-20T18:25:51Z","signature_id":"bdb0464d6ce770897024e62edf4ebdb3e733526a04930e143fcc58837d12f7c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"314b58d43728f942111a52209bcc085b0bf14adcab05fa6fc6959abcb89c83c4","failure_pattern":"dial tcp: lookup console-openshift-console","search_query":"dial tcp: lookup console-openshift-console","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":15.384615384615385,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.003s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc0008c56d0\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.f4y0j8z4x0b5v2h.j4r9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.003s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc0008c56d0\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.f4y0j8z4x0b5v2h.j4r9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.005s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc000a493b0\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.s3s6l6j1t7z6u0i.vxvo.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.005s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc000a493b0\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.s3s6l6j1t7z6u0i.vxvo.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"3f0e27144f7c16540e54da64eaf5d529650134d7ce1cb9dd58213c3bfba4fcf2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"d7dbbfa11c42b81c3ce072fc00ad80a2c5daedba03fece4d76d9a2372e748af6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-5c9rf9, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5c9rf9\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-08"],"prior_runs_affected":30,"prior_last_seen_at":"2026-06-14T23:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:126]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d52e10\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-5c9rf9, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5c9rf9\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:126]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d52e10\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-5c9rf9, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5c9rf9\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"589e3ced6c0f3551b9350e7e3c17e8f16bebdb3d9a850f58b8059bae7fb1996b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007bf3b0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007bf3b0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068216024569745408","occurred_at":"2026-06-20T06:15:00Z","signature_id":"e404ee63f95099b59271a15602d45742ef4fc421991f9967da4ec935bd09ed2c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6f26d2f1140adc9b13b39b836a7ca29a860f5c4cb3811309f672f97a11466f4d","failure_pattern":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True","search_query":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013bc3f0\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013bc3f0\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"6ea2fb9103864054f25d2a7f96b2639165b5133cc5dfde5600c28d12596644c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5fb766e5fb248dc6af1c6e24ee78b09053884360d90cdc4ef6d91ff56deaf7ff","failure_pattern":"couldn't get current server API group list: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: i/o timeout","search_query":"couldn't get current server API group list: Get \\\"https://api.h5h5i6l8r7c3p7a.u2v6.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 605.076s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:403 with:\nkubectl get clusteroperators should succeed from authorized VM\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a7a390\u003e: \n    E0621 00:06:17.087501    3391 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: Get \\\"https://api.h5h5i6l8r7c3p7a.u2v6.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout\"\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 605.076s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:403 with:\nkubectl get clusteroperators should succeed from authorized VM\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a7a390\u003e: \n    E0621 00:06:17.087501    3391 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: Get \\\"https://api.h5h5i6l8r7c3p7a.u2v6.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout\"\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"845e4a6599a7f5e04ff52f495d06347735a3f21e94149fab4f1937b471805e4e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t2dlwt\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-18"],"prior_runs_affected":4,"prior_last_seen_at":"2026-05-20T23:07:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000cf4240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t2dlwt\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000cf4240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t2dlwt\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","signature_id":"a228e83d8a237c93d67c480a8b9cadd5188b1fa5ff39e9f7307823961e0d48e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7816b6118b6d30763d9aeef2698a47954586af9058cb4051aa7efd88e0110aab","failure_pattern":"failed to create namespace: Post \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: i/o timeout","search_query":"failed to create namespace: Post \"https://api.q8q3f7q1h7j5r8s.w6pe.brazilsouth.aroapp-hcp.io:443/api/v1/namespaces\": dial tcp 74.163.139.192:443: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-14..2026-06-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-qqg5xh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1a060\u003e: \n    failed to create namespace: Post \"https://api.q8q3f7q1h7j5r8s.w6pe.brazilsouth.aroapp-hcp.io:443/api/v1/namespaces\": dial tcp 74.163.139.192:443: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-qqg5xh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1a060\u003e: \n    failed to create namespace: Post \"https://api.q8q3f7q1h7j5r8s.w6pe.brazilsouth.aroapp-hcp.io:443/api/v1/namespaces\": dial tcp 74.163.139.192:443: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","signature_id":"30859de944c389a71cc045ef67649a8410c69e8cda4db1662714ac7bf2cacbef","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}