{"meta":{"start_date":"2026-06-21","end_date":"2026-06-21","start_at":"2026-06-21T00:00:00Z","end_at":"2026-06-22T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T07:43:59Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":8,"failed_runs":6,"raw_occurrences":38,"matched_occurrences":38,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-x7v479, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-x7v479\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-x7v479, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":50,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-01","2026-06-08"],"prior_runs_affected":60,"prior_last_seen_at":"2026-06-14T23:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-tks2fl\" with version 4.22.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00138c300\u003e: \n failed to create HCP cluster cluster-zstream-4-22-tks2fl, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-tks2fl in resource group rg-zstream-upgrade-4-22-dff7ch, error: failed waiting for cluster=\"cluster-zstream-4-22-tks2fl\" in resourcegroup=\"rg-zstream-upgrade-4-22-dff7ch\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-tks2fl in resource group rg-zstream-upgrade-4-22-dff7ch, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-tks2fl\" with version 4.22.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00138c300\u003e: \n failed to create HCP cluster cluster-zstream-4-22-tks2fl, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-tks2fl in resource group rg-zstream-upgrade-4-22-dff7ch, error: failed waiting for cluster=\"cluster-zstream-4-22-tks2fl\" in resourcegroup=\"rg-zstream-upgrade-4-22-dff7ch\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-tks2fl in resource group rg-zstream-upgrade-4-22-dff7ch, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-prpz8r\" with version 4.22.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff0660\u003e: \n failed to create HCP cluster cluster-zstream-4-22-prpz8r, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-prpz8r in resource group rg-zstream-upgrade-4-22-xncgh2, error: failed waiting for cluster=\"cluster-zstream-4-22-prpz8r\" in resourcegroup=\"rg-zstream-upgrade-4-22-xncgh2\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-prpz8r in resource group rg-zstream-upgrade-4-22-xncgh2, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-prpz8r\" with version 4.22.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff0660\u003e: \n failed to create HCP cluster cluster-zstream-4-22-prpz8r, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-prpz8r in resource group rg-zstream-upgrade-4-22-xncgh2, error: failed waiting for cluster=\"cluster-zstream-4-22-prpz8r\" in resourcegroup=\"rg-zstream-upgrade-4-22-xncgh2\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-prpz8r in resource group rg-zstream-upgrade-4-22-xncgh2, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0012eec00\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-x7v479, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-x7v479\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-x7v479, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0012eec00\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-x7v479, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-x7v479\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-x7v479, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2068831445857603584","occurred_at":"2026-06-21T23:00:28Z","signature_id":"f48baf60456e8c8f23bf04b3cca673ef8e09b46475aecaa55dd4f021da24e86a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"5eeba7f9d399a4e87dc8697cbda15319d7910a25a758b506fa4e3c4e6df855cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"ee3f81af6893813c4fe65449fc6baf33f8fde524dc680b8d1b396346f24d7370","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"fcc584da3f358dafd2acb1beda5bee3eb4a510a34efe40e1b55f375ee5db46a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2068514352226373632","occurred_at":"2026-06-21T02:00:27Z","signature_id":"abd5e627204b7a0abf6b9cbdc892db64709737947315dcbc61a40a23ddefc977","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068492780056875008","occurred_at":"2026-06-21T00:34:44Z","signature_id":"7e810fb83a5343d2fd78cf22e1b59e3544c41b005fc6ab5efe6bc6aa4681bdb7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":37.5,"anchor_occurrences":6,"after_last_push_count":23,"trend_counts":[0,0,0,0,10,7,6],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-blwhnk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000886a60\u003e: \n route was never reachable: Get \"https://agnhost-e2e-serving-app-4j25v.apps.aro.f2b3g0f2g4h5f7w.mhke.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-4j25v.apps.aro.f2b3g0f2g4h5f7w.mhke.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-blwhnk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000886a60\u003e: \n route was never reachable: Get \"https://agnhost-e2e-serving-app-4j25v.apps.aro.f2b3g0f2g4h5f7w.mhke.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-4j25v.apps.aro.f2b3g0f2g4h5f7w.mhke.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-sdplf7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706300\u003e: \n route was never reachable: Get \"https://agnhost-e2e-serving-app-zxdd2.apps.aro.i9r9j8o2x4u9b9e.7lui.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-zxdd2.apps.aro.i9r9j8o2x4u9b9e.7lui.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-sdplf7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706300\u003e: \n route was never reachable: Get \"https://agnhost-e2e-serving-app-zxdd2.apps.aro.i9r9j8o2x4u9b9e.7lui.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-zxdd2.apps.aro.i9r9j8o2x4u9b9e.7lui.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pvzn6h\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000986860\u003e: \n route was never reachable: Get \"https://agnhost-e2e-serving-app-zwp5z.apps.aro.p2d4i7x8i2d6g9t.og38.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-zwp5z.apps.aro.p2d4i7x8i2d6g9t.og38.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pvzn6h\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000986860\u003e: \n route was never reachable: Get \"https://agnhost-e2e-serving-app-zwp5z.apps.aro.p2d4i7x8i2d6g9t.og38.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-zwp5z.apps.aro.p2d4i7x8i2d6g9t.og38.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"137035677c29233947781ae82c2f1afea45ad10a1e7f89b0cae8b63aff6c0d20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"d674149bd758d38cb5142522e036936a626665ef569648ecebb27ea2be2408f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"384432d3facac9dec5d66f160f8cf7277cc784030f6b220f9c8eee2a2182d2e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"d112613a80ceb50339ff7e10c69d82094f28cac759c5389d0556d132097c0051","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"2b26956777ba2cbee00b58fcc2f0780273d2e295937ee3da4a89c2db628f87d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"53dcdcda804a871cb605a6ae1a7ad2efa54e100cbca75a25e1015c682b915bd1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-5rk4ks, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5rk4ks\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":37.5,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-08"],"prior_runs_affected":33,"prior_last_seen_at":"2026-06-14T23:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:212]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001215e90\u003e: \n failed to create NodePool autoscale-noaz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-5rk4ks, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5rk4ks\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:212]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001215e90\u003e: \n failed to create NodePool autoscale-noaz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-5rk4ks, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5rk4ks\" to finish creating: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:137]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000957830\u003e: \n failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-dtvsk2, error: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dtvsk2\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:137]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000957830\u003e: \n failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-dtvsk2, error: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dtvsk2\" to finish creating: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:126]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001255950\u003e: \n failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-5rpxgq, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5rpxgq\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:126]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001255950\u003e: \n failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-5rpxgq, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5rpxgq\" to finish creating: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"0b2b6bd3551fc1f96bdc91a699aa12b5cb6bb203ece76adcc8665f77408e3ac2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"8e6a92772a1edba4420eba8b39e0728152ea57dfa648cbabf17636ec86b3b117","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"b5eb1c6e4221308e294415367c47587ca3d7638d9bfbe683561f38f8e7f7f84e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"bd026a865aaee9dbcb4869c708688dff12818f033bcc0e9e5e7f9ec607b61892","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"c54dfaab445f5493fa99c2c1be4aa7d2b8d5c4f8a76270530a408da6c3ae631f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"78754e29456c1fc88cbae908323785a403c4346735c925fc3102d3c04ce7e5df","failure_pattern":"VerifySimpleWebApp failed: route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-44c7d.apps.aro.cilium-cl.vva9.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-44c7d.apps.aro.cilium-cl.vva9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":37.5,"anchor_occurrences":4,"after_last_push_count":10,"trend_counts":[0,0,0,0,2,4,4],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001dec1b0\u003e: \n VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-g7mbq.apps.aro.cilium-cl.oryn.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-g7mbq.apps.aro.cilium-cl.oryn.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc001dec1b0\u003e: \n VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-g7mbq.apps.aro.cilium-cl.oryn.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-g7mbq.apps.aro.cilium-cl.oryn.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc000b48ab0\u003e: \n VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-qmtsc.apps.aro.cilium-cl.afaw.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-qmtsc.apps.aro.cilium-cl.afaw.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc000b48ab0\u003e: \n VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-qmtsc.apps.aro.cilium-cl.afaw.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-qmtsc.apps.aro.cilium-cl.afaw.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0011601b0\u003e: \n VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-f9l8r.apps.aro.cilium-cluster.9wlt.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-f9l8r.apps.aro.cilium-cluster.9wlt.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0011601b0\u003e: \n VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-f9l8r.apps.aro.cilium-cluster.9wlt.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-f9l8r.apps.aro.cilium-cluster.9wlt.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"adc6daf01993136e65e3ffba3a7bff212919097fa24531a5b09b4329ef804474","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"08317c9e33f61bd5028e7b07f3c77c7f7f1a8afa96f93efa774218ab326e61dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"dfa1f916cf36bb293e4cce3bedf6e8307905c9025a340c1e44068dcac8711229","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"ea1d0da39b410f7d9a13e04295191e975bd1dc5cff5e3e483f94f77b0d6213b9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":25,"anchor_occurrences":4,"after_last_push_count":29,"trend_counts":[0,0,1,0,3,2,4],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-08"],"prior_runs_affected":23,"prior_last_seen_at":"2026-06-12T19:09:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":25,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[0,0,0,0,1,2,3],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fp=0xc0017ad958 sp=0xc0017ad910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000c582a8, {0x9220320, 0xc0016f5068})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0017ad9b0 sp=0xc0017ad958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000c58008, {0x9221e00, 0xc000d99020}, 0xc0017ada18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0017ad9e8 sp=0xc0017ad9b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c58008, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc0017adc60 sp=0xc0017ad9e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c58008, {0xc0018a0000, 0x1000, 0xc0009a9080?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0017adcc8 sp=0xc0017adc60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc00184e6c0, {0xc0018a0000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc0017add28 sp=0xc0017adcc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc0012731a0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc0017add60 sp=0xc0017add28 pc=0x3084943\nbufio.(*Reader).Peek(0xc0012731a0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc0017add80 sp=0xc0017add60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc00184e6c0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0017adfc8 sp=0xc0017add80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0017adfe0 sp=0xc0017adfc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0017adfe8 sp=0xc0017adfe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 206\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 249 gp=0xc000683c00 m=nil [select]:\nruntime.gopark(0xc001247768?, 0x2?, 0x40?, 0x8?, 0xc00124775c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0012475e8 sp=0xc0012475c8 pc=0x2f1e50e\nruntime.selectgo(0xc001247768, 0xc001247758, 0xc000fb27a0?, 0x0, 0x36ecd5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc001247728 sp=0xc0012475e8 pc=0x2ef92f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x92b8658, 0xc0018b60e0}, 0x8704ddd?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc001247798 sp=0xc001247728 pc=0x36ec4b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x930a440, {0x92b8658, 0xc0018b60e0}, 0x2f1baf3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc001247a98 sp=0xc001247798 pc=0x3ded372\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x92b8658, 0xc0018b60e0}, 0xc0016f51a0, {0xc000c1d728, 0x18}, {0xc00181f620, 0x1d})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc001247c20 sp=0xc001247a98 pc=0x6c2ef65\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x92b8658?, 0xc0018b60e0?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc001247cc8 sp=0xc001247c20 pc=0x6c2e992\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x92b8658?, 0xc0018b60e0?}, 0xc0018b60f0?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc001247d20 sp=0xc001247cc8 pc=0x369fa82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x92b8658, 0xc0018b60e0}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc001247e68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc001247d98 sp=0xc001247d20 pc=0x369db2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x92b85e8, 0xc000b615e0}, 0xc0016f51a0, {0xc000c1d728, 0x18}, {0xc00181f620, 0x1d}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc001247ef0 sp=0xc001247d98 pc=0x6c2e705\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc001247f78 sp=0xc001247ef0 pc=0x6c325fe\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc001247fe0 sp=0xc001247f78 pc=0x4a4be30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001247fe8 sp=0xc001247fe0 pc=0x2f26cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 248\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax 0xca\nrbx 0x0\nrcx 0x2f28ae3\nrdx 0x0\nrdi 0xc93a4a0\nrsi 0x80\nrbp 0xc0000b5728\nrsp 0xc0000b56e0\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x0\nr13 0x1\nr14 0xc000582540\nr15 0x7\nrip 0x2f28ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nfp=0xc0017ad958 sp=0xc0017ad910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000c582a8, {0x9220320, 0xc0016f5068})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0017ad9b0 sp=0xc0017ad958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000c58008, {0x9221e00, 0xc000d99020}, 0xc0017ada18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0017ad9e8 sp=0xc0017ad9b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c58008, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc0017adc60 sp=0xc0017ad9e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c58008, {0xc0018a0000, 0x1000, 0xc0009a9080?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0017adcc8 sp=0xc0017adc60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc00184e6c0, {0xc0018a0000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc0017add28 sp=0xc0017adcc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc0012731a0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc0017add60 sp=0xc0017add28 pc=0x3084943\nbufio.(*Reader).Peek(0xc0012731a0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc0017add80 sp=0xc0017add60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc00184e6c0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0017adfc8 sp=0xc0017add80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0017adfe0 sp=0xc0017adfc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0017adfe8 sp=0xc0017adfe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 206\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 249 gp=0xc000683c00 m=nil [select]:\nruntime.gopark(0xc001247768?, 0x2?, 0x40?, 0x8?, 0xc00124775c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0012475e8 sp=0xc0012475c8 pc=0x2f1e50e\nruntime.selectgo(0xc001247768, 0xc001247758, 0xc000fb27a0?, 0x0, 0x36ecd5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc001247728 sp=0xc0012475e8 pc=0x2ef92f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x92b8658, 0xc0018b60e0}, 0x8704ddd?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc001247798 sp=0xc001247728 pc=0x36ec4b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x930a440, {0x92b8658, 0xc0018b60e0}, 0x2f1baf3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc001247a98 sp=0xc001247798 pc=0x3ded372\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x92b8658, 0xc0018b60e0}, 0xc0016f51a0, {0xc000c1d728, 0x18}, {0xc00181f620, 0x1d})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc001247c20 sp=0xc001247a98 pc=0x6c2ef65\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x92b8658?, 0xc0018b60e0?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc001247cc8 sp=0xc001247c20 pc=0x6c2e992\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x92b8658?, 0xc0018b60e0?}, 0xc0018b60f0?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc001247d20 sp=0xc001247cc8 pc=0x369fa82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x92b8658, 0xc0018b60e0}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc001247e68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc001247d98 sp=0xc001247d20 pc=0x369db2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x92b85e8, 0xc000b615e0}, 0xc0016f51a0, {0xc000c1d728, 0x18}, {0xc00181f620, 0x1d}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc001247ef0 sp=0xc001247d98 pc=0x6c2e705\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc001247f78 sp=0xc001247ef0 pc=0x6c325fe\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc001247fe0 sp=0xc001247f78 pc=0x4a4be30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001247fe8 sp=0xc001247fe0 pc=0x2f26cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 248\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax 0xca\nrbx 0x0\nrcx 0x2f28ae3\nrdx 0x0\nrdi 0xc93a4a0\nrsi 0x80\nrbp 0xc0000b5728\nrsp 0xc0000b56e0\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x0\nr13 0x1\nr14 0xc000582540\nr15 0x7\nrip 0x2f28ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","untime/select.go:351 +0x8b7 fp=0xc0000c8eb8 sp=0xc0000c8d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc0013e25a0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc0000c8fc8 sp=0xc0000c8eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc0000c8fe0 sp=0xc0000c8fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c8fe8 sp=0xc0000c8fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 109\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 193 gp=0xc000e65a40 m=nil [select]:\nruntime.gopark(0xc001384f48?, 0x2?, 0xc0?, 0x4d?, 0xc001384ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001384d78 sp=0xc001384d58 pc=0x2f1e50e\nruntime.selectgo(0xc001384f48, 0xc001384ef0, 0xc00073d980?, 0x0, 0xc000c0ab70?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc001384eb8 sp=0xc001384d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc001428240)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc001384fc8 sp=0xc001384eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc001384fe0 sp=0xc001384fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001384fe8 sp=0xc001384fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 194\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 233 gp=0xc000ac68c0 m=nil [IO wait]:\nruntime.gopark(0xc0010948e0?, 0xc0018f1788?, 0xc5?, 0xbe?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0018f1768 sp=0xc0018f1748 pc=0x2f1e50e\nruntime.netpollblock(0x2f88ad8?, 0x2eaa066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0018f17a0 sp=0xc0018f1768 pc=0x2eddfb7\ninternal/poll.runtime_pollWait(0x7f8602112c00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0018f17c0 sp=0xc0018f17a0 pc=0x2f1cda5\ninternal/poll.(*pollDesc).wait(0xc0007dd100?, 0xc001341400?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0018f17e8 sp=0xc0018f17c0 pc=0x2fab907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc0007dd100, {0xc001341400, 0x2500, 0x2500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0018f1880 sp=0xc0018f17e8 pc=0x2facd39\nnet.(*netFD).Read(0xc0007dd100, {0xc001341400?, 0xc001341400?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0018f18c8 sp=0xc0018f1880 pc=0x31712e5\nnet.(*conn).Read(0xc00100ad18, {0xc001341400?, 0x7f8600572340?, 0x7f8649d3b108?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc0018f1910 sp=0xc0018f18c8 pc=0x3183385\ncrypto/tls.(*atLeastReader).Read(0xc00113f1d0, {0xc001341400?, 0xa?, 0x859?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc0018f1958 sp=0xc0018f1910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000db5ea8, {0x9220320, 0xc00113f1d0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0018f19b0 sp=0xc0018f1958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000db5c08, {0x9221e00, 0xc00100ad18}, 0xc0018f1a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0018f19e8 sp=0xc0018f19b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000db5c08, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc0018f1c60 sp=0xc0018f19e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000db5c08, {0xc0013ce000, 0x1000, 0xc000ec2fc0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0018f1cc8 sp=0xc0018f1c60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc0013258c0, {0xc0013ce000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc0018f1d28 sp=0xc0018f1cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc000e1a420)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc0018f1d60 sp=0xc0018f1d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc000e1a420, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc0018f1d80 sp=0xc0018f1d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0013258c0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0018f1fc8 sp=0xc0018f1d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0018f1fe0 sp=0xc0018f1fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0018f1fe8 sp=0xc0018f1fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 227\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax 0xca\nrbx 0x0\nrcx 0x2f28ae3\nrdx 0x0\nrdi 0xc90cc70\nrsi 0x80\nrbp 0x7ffddd31af40\nrsp 0x7ffddd31aef8\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x0\nr13 0x1\nr14 0xc90a6e0\nr15 0x1\nrip 0x2f28ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nuntime/select.go:351 +0x8b7 fp=0xc0000c8eb8 sp=0xc0000c8d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc0013e25a0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc0000c8fc8 sp=0xc0000c8eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc0000c8fe0 sp=0xc0000c8fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c8fe8 sp=0xc0000c8fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 109\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 193 gp=0xc000e65a40 m=nil [select]:\nruntime.gopark(0xc001384f48?, 0x2?, 0xc0?, 0x4d?, 0xc001384ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001384d78 sp=0xc001384d58 pc=0x2f1e50e\nruntime.selectgo(0xc001384f48, 0xc001384ef0, 0xc00073d980?, 0x0, 0xc000c0ab70?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc001384eb8 sp=0xc001384d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc001428240)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc001384fc8 sp=0xc001384eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc001384fe0 sp=0xc001384fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001384fe8 sp=0xc001384fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 194\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 233 gp=0xc000ac68c0 m=nil [IO wait]:\nruntime.gopark(0xc0010948e0?, 0xc0018f1788?, 0xc5?, 0xbe?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0018f1768 sp=0xc0018f1748 pc=0x2f1e50e\nruntime.netpollblock(0x2f88ad8?, 0x2eaa066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0018f17a0 sp=0xc0018f1768 pc=0x2eddfb7\ninternal/poll.runtime_pollWait(0x7f8602112c00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0018f17c0 sp=0xc0018f17a0 pc=0x2f1cda5\ninternal/poll.(*pollDesc).wait(0xc0007dd100?, 0xc001341400?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0018f17e8 sp=0xc0018f17c0 pc=0x2fab907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc0007dd100, {0xc001341400, 0x2500, 0x2500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0018f1880 sp=0xc0018f17e8 pc=0x2facd39\nnet.(*netFD).Read(0xc0007dd100, {0xc001341400?, 0xc001341400?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0018f18c8 sp=0xc0018f1880 pc=0x31712e5\nnet.(*conn).Read(0xc00100ad18, {0xc001341400?, 0x7f8600572340?, 0x7f8649d3b108?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc0018f1910 sp=0xc0018f18c8 pc=0x3183385\ncrypto/tls.(*atLeastReader).Read(0xc00113f1d0, {0xc001341400?, 0xa?, 0x859?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc0018f1958 sp=0xc0018f1910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000db5ea8, {0x9220320, 0xc00113f1d0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0018f19b0 sp=0xc0018f1958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000db5c08, {0x9221e00, 0xc00100ad18}, 0xc0018f1a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0018f19e8 sp=0xc0018f19b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000db5c08, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc0018f1c60 sp=0xc0018f19e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000db5c08, {0xc0013ce000, 0x1000, 0xc000ec2fc0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0018f1cc8 sp=0xc0018f1c60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc0013258c0, {0xc0013ce000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc0018f1d28 sp=0xc0018f1cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc000e1a420)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc0018f1d60 sp=0xc0018f1d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc000e1a420, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc0018f1d80 sp=0xc0018f1d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0013258c0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0018f1fc8 sp=0xc0018f1d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0018f1fe0 sp=0xc0018f1fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0018f1fe8 sp=0xc0018f1fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 227\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax 0xca\nrbx 0x0\nrcx 0x2f28ae3\nrdx 0x0\nrdi 0xc90cc70\nrsi 0x80\nrbp 0x7ffddd31af40\nrsp 0x7ffddd31aef8\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0x0\nr13 0x1\nr14 0xc90a6e0\nr15 0x1\nrip 0x2f28ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","0f92958 sp=0xc000f92910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000fb6628, {0x9220320, 0xc0010c5ef0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000f929b0 sp=0xc000f92958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000fb6388, {0x9221e00, 0xc001114078}, 0xc000f92a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000f929e8 sp=0xc000f929b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000fb6388, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000f92c60 sp=0xc000f929e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000fb6388, {0xc0018a4000, 0x1000, 0xc00110be90?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000f92cc8 sp=0xc000f92c60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc0017dfd40, {0xc0018a4000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc000f92d28 sp=0xc000f92cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc001886c60)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc000f92d60 sp=0xc000f92d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc001886c60, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc000f92d80 sp=0xc000f92d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0017dfd40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc000f92fc8 sp=0xc000f92d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc000f92fe0 sp=0xc000f92fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000f92fe8 sp=0xc000f92fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 181\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 187 gp=0xc000bbee00 m=nil [select]:\nruntime.gopark(0xc0012a1768?, 0x2?, 0x20?, 0x18?, 0xc0012a175c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0012a15e8 sp=0xc0012a15c8 pc=0x2f1e50e\nruntime.selectgo(0xc0012a1768, 0xc0012a1758, 0xc000be0080?, 0x0, 0x36ecd5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0012a1728 sp=0xc0012a15e8 pc=0x2ef92f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x92b8658, 0xc00004cf50}, 0x8704ddd?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc0012a1798 sp=0xc0012a1728 pc=0x36ec4b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x930a440, {0x92b8658, 0xc00004cf50}, 0x2f1baf3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc0012a1a98 sp=0xc0012a1798 pc=0x3ded372\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x92b8658, 0xc00004cf50}, 0xc0014f41c8, {0xc0001732f0, 0x18}, {0xc001874940, 0xe})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc0012a1c20 sp=0xc0012a1a98 pc=0x6c2ef65\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x92b8658?, 0xc00004cf50?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc0012a1cc8 sp=0xc0012a1c20 pc=0x6c2e992\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x92b8658?, 0xc00004cf50?}, 0xc00004cf60?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc0012a1d20 sp=0xc0012a1cc8 pc=0x369fa82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x92b8658, 0xc00004cf50}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc0012a1e68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc0012a1d98 sp=0xc0012a1d20 pc=0x369db2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x92b85e8, 0xc0008411d0}, 0xc0014f41c8, {0xc0001732f0, 0x18}, {0xc001874940, 0xe}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc0012a1ef0 sp=0xc0012a1d98 pc=0x6c2e705\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc0012a1f78 sp=0xc0012a1ef0 pc=0x6c325fe\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc0012a1fe0 sp=0xc0012a1f78 pc=0x4a4be30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0012a1fe8 sp=0xc0012a1fe0 pc=0x2f26cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 176\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax 0xca\nrbx 0x0\nrcx 0x2f28ae3\nrdx 0x0\nrdi 0xc90cc70\nrsi 0x80\nrbp 0x7ffe53ff0f80\nrsp 0x7ffe53ff0f38\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0xc000096008\nr13 0x1\nr14 0xc90a6e0\nr15 0x3\nrip 0x2f28ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n0f92958 sp=0xc000f92910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000fb6628, {0x9220320, 0xc0010c5ef0})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000f929b0 sp=0xc000f92958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000fb6388, {0x9221e00, 0xc001114078}, 0xc000f92a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000f929e8 sp=0xc000f929b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000fb6388, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000f92c60 sp=0xc000f929e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000fb6388, {0xc0018a4000, 0x1000, 0xc00110be90?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000f92cc8 sp=0xc000f92c60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc0017dfd40, {0xc0018a4000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc000f92d28 sp=0xc000f92cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc001886c60)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc000f92d60 sp=0xc000f92d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc001886c60, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc000f92d80 sp=0xc000f92d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0017dfd40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc000f92fc8 sp=0xc000f92d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc000f92fe0 sp=0xc000f92fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000f92fe8 sp=0xc000f92fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 181\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 187 gp=0xc000bbee00 m=nil [select]:\nruntime.gopark(0xc0012a1768?, 0x2?, 0x20?, 0x18?, 0xc0012a175c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0012a15e8 sp=0xc0012a15c8 pc=0x2f1e50e\nruntime.selectgo(0xc0012a1768, 0xc0012a1758, 0xc000be0080?, 0x0, 0x36ecd5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0012a1728 sp=0xc0012a15e8 pc=0x2ef92f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x92b8658, 0xc00004cf50}, 0x8704ddd?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc0012a1798 sp=0xc0012a1728 pc=0x36ec4b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x930a440, {0x92b8658, 0xc00004cf50}, 0x2f1baf3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc0012a1a98 sp=0xc0012a1798 pc=0x3ded372\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x92b8658, 0xc00004cf50}, 0xc0014f41c8, {0xc0001732f0, 0x18}, {0xc001874940, 0xe})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc0012a1c20 sp=0xc0012a1a98 pc=0x6c2ef65\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x92b8658?, 0xc00004cf50?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc0012a1cc8 sp=0xc0012a1c20 pc=0x6c2e992\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x92b8658?, 0xc00004cf50?}, 0xc00004cf60?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc0012a1d20 sp=0xc0012a1cc8 pc=0x369fa82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x92b8658, 0xc00004cf50}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc0012a1e68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc0012a1d98 sp=0xc0012a1d20 pc=0x369db2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x92b85e8, 0xc0008411d0}, 0xc0014f41c8, {0xc0001732f0, 0x18}, {0xc001874940, 0xe}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc0012a1ef0 sp=0xc0012a1d98 pc=0x6c2e705\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc0012a1f78 sp=0xc0012a1ef0 pc=0x6c325fe\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc0012a1fe0 sp=0xc0012a1f78 pc=0x4a4be30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0012a1fe8 sp=0xc0012a1fe0 pc=0x2f26cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 176\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax 0xca\nrbx 0x0\nrcx 0x2f28ae3\nrdx 0x0\nrdi 0xc90cc70\nrsi 0x80\nrbp 0x7ffe53ff0f80\nrsp 0x7ffe53ff0f38\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0xc000096008\nr13 0x1\nr14 0xc90a6e0\nr15 0x3\nrip 0x2f28ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"39e31a45f7893323e79497d4ba9e88011f07c0bf4fa36c1b7def69fa3b6e1864","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"f687d05df23945fcfcd89fffacf3c318a04b5327ab0ad9d3dd8287b33c5d34f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"edfc7d171d9432f33616b5e78ff57a5552c370e3edbcbc853da769f99419439e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b286b88e4cdf2beb25fd0ed17c671477745b97698c63efca626d18c2b4af0e4","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,0,2],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:406 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:406 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:406 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 600.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:406 with:\nAll ClusterOperators should report Available=True, but these are not available: [console (Available=False) ingress (Available=False)]\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"8a51e3752f4ecefbc99ac8fb256989896f814927f6f7b0213f01f8698f9b93e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"1e54c4d2a6823549b2cd272a44fa8959e87befde1fe048334f2942b983950a8e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"314b58d43728f942111a52209bcc085b0bf14adcab05fa6fc6959abcb89c83c4","failure_pattern":"dial tcp: lookup console-openshift-console","search_query":"dial tcp: lookup console-openshift-console","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,0,0,1,2,2],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003c*net.OpError | 0xc000bdaf00\u003e: \n dial tcp: lookup console-openshift-console.apps.aro.i9f1o4d9p7n0g1u.dh5j.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003c*net.OpError | 0xc000bdaf00\u003e: \n dial tcp: lookup console-openshift-console.apps.aro.i9f1o4d9p7n0g1u.dh5j.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003c*net.OpError | 0xc000b7eb40\u003e: \n dial tcp: lookup console-openshift-console.apps.aro.m1c4h3t4a2g8i8d.4hwy.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n \u003c*net.OpError | 0xc000b7eb40\u003e: \n dial tcp: lookup console-openshift-console.apps.aro.m1c4h3t4a2g8i8d.4hwy.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"295fa52bfa7c29f1273219114013714cab96c77346f7a60513f99715ca57e550","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"65ce058b6b36a04086262d68a3f05794145651f7ad64fc3e882e74858191ab8f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"eaff0beaf9dac327fa146f686a15a3b9b0e697c496cc191c47b118fc7c2ca7c3","failure_pattern":"ERROR CODE: InternalServerError; detail message Get \"\u003curl\u003e\": net/http: TLS handshake timeout; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009535a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-sfdjsl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9494c8db-21a8-45dc-a144-9909ebdbc9f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9494c8db-21a8-45dc-a144-9909ebdbc9f1\",\n \"name\": \"9494c8db-21a8-45dc-a144-9909ebdbc9f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-06-21T07:22:15.977086685Z\",\n \"endTime\": \"2026-06-21T07:23:35.108555439Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Get \\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus?%24filter=location+eq+%27brazilsouth%27\\u0026api-version=2021-07-01\\\": net/http: TLS handshake timeout\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009535a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-sfdjsl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9494c8db-21a8-45dc-a144-9909ebdbc9f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9494c8db-21a8-45dc-a144-9909ebdbc9f1\",\n \"name\": \"9494c8db-21a8-45dc-a144-9909ebdbc9f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-06-21T07:22:15.977086685Z\",\n \"endTime\": \"2026-06-21T07:23:35.108555439Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Get \\\"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus?%24filter=location+eq+%27brazilsouth%27\\u0026api-version=2021-07-01\\\": net/http: TLS handshake timeout\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"3156eb4cb3e78571e4278eaf2fc4b5c131d0ba36d5c18676d4e21b99406283fb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"875cdf078d519e224c878ffcb1064625162cc0e07added949f960e01232396ca","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v4.20.0-0.nightly-multi-2026-06-21-183852-nightly' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86220\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-s2fhzz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-s2fhzz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.20.0-0.nightly-multi-2026-06-21-183852-nightly' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86220\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-s2fhzz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-s2fhzz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Version 'openshift-v4.20.0-0.nightly-multi-2026-06-21-183852-nightly' doesn't exist\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2068831445857603584","occurred_at":"2026-06-21T23:00:28Z","signature_id":"feeb177d17c5274b994d1e4d66dd0ba9cc0e1d94d40a17a6d18952d9f9af0f5c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000e89980\u003e: \n add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000e89980\u003e: \n add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068520879767162880","occurred_at":"2026-06-21T02:26:23Z","signature_id":"c1f9a56242ead2f7e1e1803e07d603245b31a7192f786cb433ba7eb75405f0d5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[1,2,0,1,1,2,1],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-18","2026-06-08"],"prior_runs_affected":12,"prior_last_seen_at":"2026-06-13T23:00:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0xc001090000\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0xc001090000\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068591174670618624","occurred_at":"2026-06-21T07:05:43Z","signature_id":"5b1ac5edbe9bb5b6d6586f1a9226cbcd1556ce2a271fa0584fe318f8b9628e00","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5fb766e5fb248dc6af1c6e24ee78b09053884360d90cdc4ef6d91ff56deaf7ff","failure_pattern":"couldn't get current server API group list: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: i/o timeout","search_query":"couldn't get current server API group list: Get \\\"https://api.s6u4e9h6n4x5p4f.go5d.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 600.008s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:403 with:\nkubectl get clusteroperators should succeed from authorized VM\nUnexpected error:\n \u003c*errors.errorString | 0xc001703260\u003e: \n E0621 11:01:15.116862 3256 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: Get \\\"https://api.s6u4e9h6n4x5p4f.go5d.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout\"\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 600.008s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:403 with:\nkubectl get clusteroperators should succeed from authorized VM\nUnexpected error:\n \u003c*errors.errorString | 0xc001703260\u003e: \n E0621 11:01:15.116862 3256 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: Get \\\"https://api.s6u4e9h6n4x5p4f.go5d.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout\"\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"df5257332329d21841cb94d5779c9040ab949bb5e45243f1bbbb5d2b1d9b7e3d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gj9qbc\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":12,"trend_counts":[0,0,2,0,8,1,1],"trend_range":"2026-06-15..2026-06-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-18"],"prior_runs_affected":5,"prior_last_seen_at":"2026-05-20T23:07:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fea4b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gj9qbc\" to finish deleting: context canceled\n ...\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000fea4b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-gj9qbc\" to finish deleting: context canceled\n ...\noccurred \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068641277968125952","occurred_at":"2026-06-21T10:24:48Z","signature_id":"3c0dbbfa33e0ecd8e68284b65e9d3ef2a317e94df1734a79040d10ac52751ce6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}