{"meta":{"date":"2026-06-20","timezone":"UTC","generated_at":"2026-08-29T16:02:51Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":13,"failed_runs":10,"runs_with_occurrences":10,"runs_with_failure_pattern_matches":9,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2068469052392607744","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T23:00:27Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"50d80daf6f1879d6b2bd417d2d08c3ef717121e367d730564ed29a84ab7d484a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2068469052392607744","occurred_at":"2026-06-20T23:00:27Z","failed_at":"e2e","signature_id":"8d531303f114240ae47837d3f93a8dd60e92225d821d454820c92e5a5ba27fc1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-lvjsdl/cluster-nightly-5-0-hxzbgf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016204e0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-hxzbgf, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-hxzbgf in resource group rg-nightly-5-0-lvjsdl, error: failed waiting for cluster=\"cluster-nightly-5-0-hxzbgf\" in resourcegroup=\"rg-nightly-5-0-lvjsdl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-hxzbgf in resource group rg-nightly-5-0-lvjsdl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-lvjsdl/cluster-nightly-5-0-hxzbgf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016204e0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-hxzbgf, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-hxzbgf in resource group rg-nightly-5-0-lvjsdl, error: failed waiting for cluster=\"cluster-nightly-5-0-hxzbgf\" in resourcegroup=\"rg-nightly-5-0-lvjsdl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-hxzbgf in resource group rg-nightly-5-0-lvjsdl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded"}},{"row_id":"ca49dfe34d13f92a25934b0012c22be4bb476ebd17d13034eff4eae9d489c46a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2068469052392607744","occurred_at":"2026-06-20T23:00:27Z","failed_at":"e2e","signature_id":"1374440a9b7fdf0dd4c5846fb1575b98eb5b806accbce15929c0e84a45cc26e5","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-55jt6c\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00146eb10\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-55jt6c, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-55jt6c in resource group rg-zstream-upgrade-4-22-gmxf8f, error: failed waiting for cluster=\"cluster-zstream-4-22-55jt6c\" in resourcegroup=\"rg-zstream-upgrade-4-22-gmxf8f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-55jt6c in resource group rg-zstream-upgrade-4-22-gmxf8f, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-55jt6c\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00146eb10\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-55jt6c, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-55jt6c in resource group rg-zstream-upgrade-4-22-gmxf8f, error: failed waiting for cluster=\"cluster-zstream-4-22-55jt6c\" in resourcegroup=\"rg-zstream-upgrade-4-22-gmxf8f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-55jt6c in resource group rg-zstream-upgrade-4-22-gmxf8f, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T22:44:23Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"0288858c3d98bfb61dfa1e8a44162db1f5bcd0b71c03846ca09b5dfa52ef719f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"a228e83d8a237c93d67c480a8b9cadd5188b1fa5ff39e9f7307823961e0d48e4","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000cf4240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t2dlwt\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000cf4240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t2dlwt\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-t2dlwt\" to finish deleting: context canceled"}},{"row_id":"0bc614ad23cbdc686fec960412ba20d3f4e0b0888d4a173587d18f2479d52004","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"3f0e27144f7c16540e54da64eaf5d529650134d7ce1cb9dd58213c3bfba4fcf2","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.003s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc0008c56d0\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.f4y0j8z4x0b5v2h.j4r9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.003s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc0008c56d0\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.f4y0j8z4x0b5v2h.j4r9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"314b58d43728f942111a52209bcc085b0bf14adcab05fa6fc6959abcb89c83c4","failure_pattern":"dial tcp: lookup console-openshift-console","search_query":"dial tcp: lookup console-openshift-console"}},{"row_id":"1e1f4967dfe9a378ce48155b67873f7b84613101c4a3bccf7cd1b5c09c2b0177","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"845e4a6599a7f5e04ff52f495d06347735a3f21e94149fab4f1937b471805e4e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 605.076s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:403 with:\nkubectl get clusteroperators should succeed from authorized VM\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a7a390\u003e: \n    E0621 00:06:17.087501    3391 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: Get \\\"https://api.h5h5i6l8r7c3p7a.u2v6.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout\"\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: Timed out after 605.076s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:403 with:\nkubectl get clusteroperators should succeed from authorized VM\nUnexpected error:\n    \u003c*errors.errorString | 0xc000a7a390\u003e: \n    E0621 00:06:17.087501    3391 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: Get \\\"https://api.h5h5i6l8r7c3p7a.u2v6.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout\"\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5fb766e5fb248dc6af1c6e24ee78b09053884360d90cdc4ef6d91ff56deaf7ff","failure_pattern":"couldn't get current server API group list: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: i/o timeout","search_query":"couldn't get current server API group list: Get \\\"https://api.h5h5i6l8r7c3p7a.u2v6.brazilsouth.aroapp-hcp.io:443/api?timeout=32s\\\": dial tcp 74.163.139.192:443: i/o timeout"}},{"row_id":"21d1fc50188703c443e4db2fe6931f99816d6ecb4acd4207de1a2788f0b2ec6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"f34d6af338b8454dc2204b005de839554fce4302748615240ec6a748683432ad","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-zf6pql\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00096e780\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-dvdt9.apps.aro.t9r5s9t0c4l3k0a.37yh.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-dvdt9.apps.aro.t9r5s9t0c4l3k0a.37yh.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-zf6pql\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00096e780\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-dvdt9.apps.aro.t9r5s9t0c4l3k0a.37yh.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-dvdt9.apps.aro.t9r5s9t0c4l3k0a.37yh.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred"}},{"row_id":"6772fbe0097b7f5b937ecf8495aa293dfdb952ff8f47914820ab1a862c40a2c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"5ae9f4af0de562607c311fca891af1f286068eb5c6634c83d40b2860e29f8a3f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-wxppcl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00012ae20\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-4d2pm.apps.aro.f2j5d7e4q4x3n6o.g26p.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-4d2pm.apps.aro.f2j5d7e4q4x3n6o.g26p.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-wxppcl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00012ae20\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-4d2pm.apps.aro.f2j5d7e4q4x3n6o.g26p.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-4d2pm.apps.aro.f2j5d7e4q4x3n6o.g26p.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred"}},{"row_id":"71992cd148b6a3f070c1e21e27f3bb12cef41d58d886154e87154b663104be6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"3f80849ae13bf039b50b9f059c7ba4fb1f749b1f47f6ac1f11ee3c7ab6e3815b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-lklm69/cluster-candidate-4-22-f4gmfq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014d4690\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-lklm69/cluster-candidate-4-22-f4gmfq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014d4690\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded"}},{"row_id":"ab56f5ffecc48dd091ff8c1ff1ddb1c782ede0c4185e6067940dbc0ba69a6dab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"010b43b1e276924b59a68771f3466bc2f1fc8a5a375e7d1e7657e0f0c867c176","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to verify simple web app on back-level cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f540e0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-zgg49.apps.aro.h8q7w4u9x7c7g6v.x1ie.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-zgg49.apps.aro.h8q7w4u9x7c7g6v.x1ie.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to verify simple web app on back-level cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f540e0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-zgg49.apps.aro.h8q7w4u9x7c7g6v.x1ie.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-zgg49.apps.aro.h8q7w4u9x7c7g6v.x1ie.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred"}},{"row_id":"b817ba40d871767bd9b204f119554c7c67fd661c873c7dd62443034790ac05c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"a04fb3f7f155acd62abe2cf3dddc4d784297767710d690469fb9a348af3ab4a1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-6z4flh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000954aa0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-dg7r4.apps.aro.k4f9f7h5q5s9h1o.o4cb.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-dg7r4.apps.aro.k4f9f7h5q5s9h1o.o4cb.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-6z4flh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000954aa0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-dg7r4.apps.aro.k4f9f7h5q5s9h1o.o4cb.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-dg7r4.apps.aro.k4f9f7h5q5s9h1o.o4cb.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred"}},{"row_id":"d8551c36310ca1c443077139454ad8146d0ff66a21d7005c35ecdc96a9ab35a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"495f7bdf204bbd214858933af31f5d044a5bcb1b8b501916ba253c491daaf0a1","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012542a0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-j5bdd.apps.aro.cilium-cl.17y9.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-j5bdd.apps.aro.cilium-cl.17y9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012542a0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-j5bdd.apps.aro.cilium-cl.17y9.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-j5bdd.apps.aro.cilium-cl.17y9.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"78754e29456c1fc88cbae908323785a403c4346735c925fc3102d3c04ce7e5df","failure_pattern":"VerifySimpleWebApp failed: route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host"}},{"row_id":"d8c03779df11e4392fac96855762377ac6692f9a839ff6c1b29ccbd7a16fc217","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"19a8c1da69ee46dc328267e29bc50e0a088d6a1ecdc3d076590d7368d6a3a662","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0021b6000\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-8872n.apps.aro.cilium-cluster.moy3.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-8872n.apps.aro.cilium-cluster.moy3.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0021b6000\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-8872n.apps.aro.cilium-cluster.moy3.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-8872n.apps.aro.cilium-cluster.moy3.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"78754e29456c1fc88cbae908323785a403c4346735c925fc3102d3c04ce7e5df","failure_pattern":"VerifySimpleWebApp failed: route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host"}},{"row_id":"e2ba6505a299ae9aa8421f0aa3737d49422480ff95e42a7933f1124678120e54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"6ea2fb9103864054f25d2a7f96b2639165b5133cc5dfde5600c28d12596644c8","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013bc3f0\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013bc3f0\u003e: \n    context deadline exceeded\n    cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f26d2f1140adc9b13b39b836a7ca29a860f5c4cb3811309f672f97a11466f4d","failure_pattern":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True","search_query":"cluster operators not available: console (Degraded=True, Progressing=False, Available=False, Upgradeable=True, EvaluationConditionsDetected=Unknown), ingress (Available=False, Progressing=True, Degraded=True, Upgradeable=True, EvaluationConditionsDetected=False)"}},{"row_id":"ff072c9cfac504aa5ea7e70c45785fa01f5b8f2dd2abba68122900ebc2be7c1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068465010476060672","occurred_at":"2026-06-20T22:44:23Z","failed_at":"e2e","signature_id":"ea6368dc87e56ff73bb14bc32d785a4af01fd962c88192375df0b5e51b063f4a","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"go:2125 +0x47 fp=0xc000be4d28 sp=0xc000be4cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc00162c1e0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc000be4d60 sp=0xc000be4d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc00162c1e0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc000be4d80 sp=0xc000be4d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc001359d40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc000be4fc8 sp=0xc000be4d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc000be4fe0 sp=0xc000be4fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000be4fe8 sp=0xc000be4fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 173\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 189 gp=0xc001001dc0 m=nil [select, 1 minutes]:\nruntime.gopark(0xc0016b3f48?, 0x2?, 0xc0?, 0x3d?, 0xc0016b3ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0016b3d78 sp=0xc0016b3d58 pc=0x2f1e50e\nruntime.selectgo(0xc0016b3f48, 0xc0016b3ef0, 0xc000c97240?, 0x0, 0xc0018e37d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0016b3eb8 sp=0xc0016b3d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc001359d40)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc0016b3fc8 sp=0xc0016b3eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc0016b3fe0 sp=0xc0016b3fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0016b3fe8 sp=0xc0016b3fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 173\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 235 gp=0xc0016a4fc0 m=nil [select]:\nruntime.gopark(0xc0013df768?, 0x2?, 0x90?, 0x1f?, 0xc0013df75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0013df5e8 sp=0xc0013df5c8 pc=0x2f1e50e\nruntime.selectgo(0xc0013df768, 0xc0013df758, 0xc000f5d760?, 0x0, 0x36ecd5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0013df728 sp=0xc0013df5e8 pc=0x2ef92f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x92b8658, 0xc0018e0b60}, 0x8704ddd?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc0013df798 sp=0xc0013df728 pc=0x36ec4b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x930a440, {0x92b8658, 0xc0018e0b60}, 0x2f1baf3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc0013dfa98 sp=0xc0013df798 pc=0x3ded372\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x92b8658, 0xc0018e0b60}, 0xc001037308, {0xc0010125e8, 0x12}, {0xc001546108, 0x12})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc0013dfc20 sp=0xc0013dfa98 pc=0x6c2ef65\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x92b8658?, 0xc0018e0b60?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc0013dfcc8 sp=0xc0013dfc20 pc=0x6c2e992\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x92b8658?, 0xc0018e0b60?}, 0xc0018e0b70?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc0013dfd20 sp=0xc0013dfcc8 pc=0x369fa82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x92b8658, 0xc0018e0b60}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc0013dfe68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc0013dfd98 sp=0xc0013dfd20 pc=0x369db2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x92b85e8, 0xc0008bb540}, 0xc001037308, {0xc0010125e8, 0x12}, {0xc001546108, 0x12}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc0013dfef0 sp=0xc0013dfd98 pc=0x6c2e705\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc0013dff78 sp=0xc0013dfef0 pc=0x6c325fe\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc0013dffe0 sp=0xc0013dff78 pc=0x4a4be30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0013dffe8 sp=0xc0013dffe0 pc=0x2f26cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 214\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x2f28ae3\nrdx    0x0\nrdi    0xc90cc70\nrsi    0x80\nrbp    0x7ffcdfaaa7b0\nrsp    0x7ffcdfaaa768\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x1c\nr13    0x1\nr14    0xc90a6e0\nr15    0x7\nrip    0x2f28ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\ngo:2125 +0x47 fp=0xc000be4d28 sp=0xc000be4cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc00162c1e0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc000be4d60 sp=0xc000be4d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc00162c1e0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc000be4d80 sp=0xc000be4d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc001359d40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc000be4fc8 sp=0xc000be4d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc000be4fe0 sp=0xc000be4fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000be4fe8 sp=0xc000be4fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 173\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 189 gp=0xc001001dc0 m=nil [select, 1 minutes]:\nruntime.gopark(0xc0016b3f48?, 0x2?, 0xc0?, 0x3d?, 0xc0016b3ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0016b3d78 sp=0xc0016b3d58 pc=0x2f1e50e\nruntime.selectgo(0xc0016b3f48, 0xc0016b3ef0, 0xc000c97240?, 0x0, 0xc0018e37d0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0016b3eb8 sp=0xc0016b3d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc001359d40)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc0016b3fc8 sp=0xc0016b3eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc0016b3fe0 sp=0xc0016b3fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0016b3fe8 sp=0xc0016b3fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 173\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 235 gp=0xc0016a4fc0 m=nil [select]:\nruntime.gopark(0xc0013df768?, 0x2?, 0x90?, 0x1f?, 0xc0013df75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0013df5e8 sp=0xc0013df5c8 pc=0x2f1e50e\nruntime.selectgo(0xc0013df768, 0xc0013df758, 0xc000f5d760?, 0x0, 0x36ecd5c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc0013df728 sp=0xc0013df5e8 pc=0x2ef92f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x92b8658, 0xc0018e0b60}, 0x8704ddd?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc0013df798 sp=0xc0013df728 pc=0x36ec4b6\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x930a440, {0x92b8658, 0xc0018e0b60}, 0x2f1baf3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc0013dfa98 sp=0xc0013df798 pc=0x3ded372\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x92b8658, 0xc0018e0b60}, 0xc001037308, {0xc0010125e8, 0x12}, {0xc001546108, 0x12})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc0013dfc20 sp=0xc0013dfa98 pc=0x6c2ef65\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x92b8658?, 0xc0018e0b60?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc0013dfcc8 sp=0xc0013dfc20 pc=0x6c2e992\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x92b8658?, 0xc0018e0b60?}, 0xc0018e0b70?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc0013dfd20 sp=0xc0013dfcc8 pc=0x369fa82\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x92b8658, 0xc0018e0b60}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc0013dfe68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc0013dfd98 sp=0xc0013dfd20 pc=0x369db2e\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x92b85e8, 0xc0008bb540}, 0xc001037308, {0xc0010125e8, 0x12}, {0xc001546108, 0x12}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc0013dfef0 sp=0xc0013dfd98 pc=0x6c2e705\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc0013dff78 sp=0xc0013dfef0 pc=0x6c325fe\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc0013dffe0 sp=0xc0013dff78 pc=0x4a4be30\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0013dffe8 sp=0xc0013dffe0 pc=0x2f26cc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 214\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x2f28ae3\nrdx    0x0\nrdi    0xc90cc70\nrsi    0x80\nrbp    0x7ffcdfaaa7b0\nrsp    0x7ffcdfaaa768\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x1c\nr13    0x1\nr14    0xc90a6e0\nr15    0x7\nrip    0x2f28ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","314b58d43728f942111a52209bcc085b0bf14adcab05fa6fc6959abcb89c83c4","5fb766e5fb248dc6af1c6e24ee78b09053884360d90cdc4ef6d91ff56deaf7ff","6f26d2f1140adc9b13b39b836a7ca29a860f5c4cb3811309f672f97a11466f4d","78754e29456c1fc88cbae908323785a403c4346735c925fc3102d3c04ce7e5df","a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068443878926585856","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T21:20:25Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"32134442f8d070eae248c2413f75a2cd62a166908bdd9259071491eefb80d28f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068443878926585856","occurred_at":"2026-06-20T21:20:25Z","failed_at":"e2e","signature_id":"17d655e6d8990c66981568b9fe20a63447579ff5bad898d4b6ec61e35a8a9888","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-s6l4jx/cluster-candidate-4-22-gqscd7 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a36480\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-gqscd7, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-gqscd7 in resource group rg-candidate-4-22-s6l4jx, error: failed waiting for cluster=\"cluster-candidate-4-22-gqscd7\" in resourcegroup=\"rg-candidate-4-22-s6l4jx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-gqscd7 in resource group rg-candidate-4-22-s6l4jx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-s6l4jx/cluster-candidate-4-22-gqscd7 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a36480\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-gqscd7, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-gqscd7 in resource group rg-candidate-4-22-s6l4jx, error: failed waiting for cluster=\"cluster-candidate-4-22-gqscd7\" in resourcegroup=\"rg-candidate-4-22-s6l4jx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-gqscd7 in resource group rg-candidate-4-22-s6l4jx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded"}},{"row_id":"d7d51b87dd040a08d595b60bd25d3118c674995f455ffdec1575fbdbb1e52aef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068443878926585856","occurred_at":"2026-06-20T21:20:25Z","failed_at":"e2e","signature_id":"5a0423b3923dc89a368332a2790ff39707801c91b57f2adb24e38d2edda0fd61","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-wfbfm6\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a3ca80\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-wfbfm6, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-wfbfm6 in resource group rg-zstream-upgrade-4-22-lwj2vb, error: failed waiting for cluster=\"cluster-zstream-4-22-wfbfm6\" in resourcegroup=\"rg-zstream-upgrade-4-22-lwj2vb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-wfbfm6 in resource group rg-zstream-upgrade-4-22-lwj2vb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-wfbfm6\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a3ca80\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-wfbfm6, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-wfbfm6 in resource group rg-zstream-upgrade-4-22-lwj2vb, error: failed waiting for cluster=\"cluster-zstream-4-22-wfbfm6\" in resourcegroup=\"rg-zstream-upgrade-4-22-lwj2vb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-wfbfm6 in resource group rg-zstream-upgrade-4-22-lwj2vb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068432057893130240","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T20:33:26Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b7f99e3e3a4976464a704e91cc0fa992e0d4c867605beed2156693c8dc4aa47e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068432057893130240","occurred_at":"2026-06-20T20:33:26Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T18:35:22Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"114fb34824fc48d60781c53a981b5e727cc5ecc7282aeed0d6da7df7cf270944","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"4c0bb84b47552cb6c935e3c0563b31ab7117a9c9fbe4151088f52951f0cdf408","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-rlb9dn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e98460\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-xnczv.apps.aro.w3y9b1h7k1a0m3a.ntaj.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-xnczv.apps.aro.w3y9b1h7k1a0m3a.ntaj.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-rlb9dn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e98460\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-xnczv.apps.aro.w3y9b1h7k1a0m3a.ntaj.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-xnczv.apps.aro.w3y9b1h7k1a0m3a.ntaj.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred"}},{"row_id":"211220cb4938bffe481bd90c21e7a478423a826af215a7e6dc7ce950d8b798c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"1111515492f30e2408acccb3968c1c03c551b248e3cde5b1657318baf89edc91","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011fe378\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011fe378\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"78754e29456c1fc88cbae908323785a403c4346735c925fc3102d3c04ce7e5df","failure_pattern":"VerifySimpleWebApp failed: route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host"}},{"row_id":"221a1e8f021cd9b1816236462aac12c8e580f91ddcc36dabba9ff553fcb34e0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"d7dbbfa11c42b81c3ce072fc00ad80a2c5daedba03fece4d76d9a2372e748af6","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.005s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc000a493b0\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.s3s6l6j1t7z6u0i.vxvo.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.005s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc000a493b0\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.s3s6l6j1t7z6u0i.vxvo.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"314b58d43728f942111a52209bcc085b0bf14adcab05fa6fc6959abcb89c83c4","failure_pattern":"dial tcp: lookup console-openshift-console","search_query":"dial tcp: lookup console-openshift-console"}},{"row_id":"362520fcc8801e5fdf19ee3fc67f227fa96bb72a5e5d67ce39f3cfad118ad0b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"61588617151eb55006027d519d5dbf47bf1b0f513822a4897fbfdb0dac1b1198","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc00160ea98\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:244]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0xc00160ea98\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"5d6adab15bc9d66f3e05c77b8eeed6f2eb2113a1762ae8272b89a1431bd8b684","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"7e708cce8624ecb4df2576f7d6e3253a9bfce775ae2d53fe9470231ef30ada56","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pbdr47\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e90800\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-gtxbf.apps.aro.c0d5y7d7l4p2v7s.4c6c.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-gtxbf.apps.aro.c0d5y7d7l4p2v7s.4c6c.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-pbdr47\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e90800\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-gtxbf.apps.aro.c0d5y7d7l4p2v7s.4c6c.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-gtxbf.apps.aro.c0d5y7d7l4p2v7s.4c6c.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred"}},{"row_id":"7f9f293804f5f272f0d5d009e624edb16a58f59649666112f90b00a05936b2f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"30859de944c389a71cc045ef67649a8410c69e8cda4db1662714ac7bf2cacbef","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-qqg5xh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1a060\u003e: \n    failed to create namespace: Post \"https://api.q8q3f7q1h7j5r8s.w6pe.brazilsouth.aroapp-hcp.io:443/api/v1/namespaces\": dial tcp 74.163.139.192:443: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-qqg5xh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1a060\u003e: \n    failed to create namespace: Post \"https://api.q8q3f7q1h7j5r8s.w6pe.brazilsouth.aroapp-hcp.io:443/api/v1/namespaces\": dial tcp 74.163.139.192:443: i/o timeout\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7816b6118b6d30763d9aeef2698a47954586af9058cb4051aa7efd88e0110aab","failure_pattern":"failed to create namespace: Post \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: i/o timeout","search_query":"failed to create namespace: Post \"https://api.q8q3f7q1h7j5r8s.w6pe.brazilsouth.aroapp-hcp.io:443/api/v1/namespaces\": dial tcp 74.163.139.192:443: i/o timeout"}},{"row_id":"9b07eff47a73cf1b6aec7791f3162b933eec1b3e6efcf0add1594fff26252074","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"2c746df90ce47ce86f765b37e298a40847ec8905f8537e45167732ed3aa8f5f5","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to verify simple web app on back-level cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f2e040\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-d7rns.apps.aro.f8w9v6v4z8w1e2e.bggi.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-d7rns.apps.aro.f8w9v6v4z8w1e2e.bggi.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to verify simple web app on back-level cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f2e040\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-d7rns.apps.aro.f8w9v6v4z8w1e2e.bggi.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-d7rns.apps.aro.f8w9v6v4z8w1e2e.bggi.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","failure_pattern":"route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"occurred"}},{"row_id":"a300878bd4a57d119502e02b23381f0afef48d74663af5ea9216dd56659f411b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"0275320401f1ba8581c52de46c654704d3f59df8b34a9f21dacf93ee711e9816","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:95]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001502f30\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-knlcxx, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-knlcxx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-knlcxx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:95]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001502f30\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-knlcxx, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-knlcxx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-knlcxx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded"}},{"row_id":"a5cff0c7e318f460e2c1982ce8703eb1d1503d8915a6e5cc1fd00b29d0bdbf7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"589e3ced6c0f3551b9350e7e3c17e8f16bebdb3d9a850f58b8059bae7fb1996b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:126]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d52e10\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-5c9rf9, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5c9rf9\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:126]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d52e10\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-5c9rf9, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5c9rf9\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-5c9rf9, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5c9rf9\" to finish creating: context deadline exceeded"}},{"row_id":"d5281d9fb8972420407a9448904b26659398bc2d0fcd212833696991bc8f1969","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"db06cbe51b9df6e21a0c1a14a7ac993df73473318c3c6eeb92a1894e9d530650","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"cc133a77418a5acbe06aa2926952a966fa6b5a5c0d00dc0d96988b71409fe672","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e23290\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-rwks6.apps.aro.cilium-cl.23l1.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-rwks6.apps.aro.cilium-cl.23l1.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e23290\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-rwks6.apps.aro.cilium-cl.23l1.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-rwks6.apps.aro.cilium-cl.23l1.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"78754e29456c1fc88cbae908323785a403c4346735c925fc3102d3c04ce7e5df","failure_pattern":"VerifySimpleWebApp failed: route was never reachable: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io\": dial tcp: lookup agnhost-e2e-serving-app-6tkcv.apps.aro.cilium-cluster.aq51.brazilsouth.aroapp-hcp.io on 172.30.0.10:53: no such host"}},{"row_id":"f714de9f2475bb497374ac3dbab71ed74edabbe091d7318eb5322afd24f985ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068402345783332864","occurred_at":"2026-06-20T18:35:22Z","failed_at":"e2e","signature_id":"12615c31a81b23d87c528a95f1eee5e33503c3eea5de7e32d04e9c7147e5f757","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"53 fp=0xc0000c9d80 sp=0xc0000c9d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0010a6b40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0000c9fc8 sp=0xc0000c9d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0000c9fe0 sp=0xc0000c9fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c9fe8 sp=0xc0000c9fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 212\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 236 gp=0xc000e1ec40 m=nil [select, 2 minutes]:\nruntime.gopark(0xc001303f48?, 0x2?, 0xc0?, 0x3d?, 0xc001303ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000382d78 sp=0xc000382d58 pc=0x2f1e50e\nruntime.selectgo(0xc000382f48, 0xc001303ef0, 0xc0008e8b40?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000382eb8 sp=0xc000382d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc0018445a0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000382fc8 sp=0xc000382eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000382fe0 sp=0xc000382fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000382fe8 sp=0xc000382fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 197\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 235 gp=0xc000682e00 m=nil [IO wait, 2 minutes]:\nruntime.gopark(0xc0000d5760?, 0xc001105788?, 0xc5?, 0xbe?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001105768 sp=0xc001105748 pc=0x2f1e50e\nruntime.netpollblock(0x2f88ad8?, 0x2eaa066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0011057a0 sp=0xc001105768 pc=0x2eddfb7\ninternal/poll.runtime_pollWait(0x7f51f5ba3e00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0011057c0 sp=0xc0011057a0 pc=0x2f1cda5\ninternal/poll.(*pollDesc).wait(0xc001866680?, 0xc001728a00?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0011057e8 sp=0xc0011057c0 pc=0x2fab907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc001866680, {0xc001728a00, 0x2500, 0x2500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc001105880 sp=0xc0011057e8 pc=0x2facd39\nnet.(*netFD).Read(0xc001866680, {0xc001728a00?, 0xc001728a00?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0011058c8 sp=0xc001105880 pc=0x31712e5\nnet.(*conn).Read(0xc000d6aa10, {0xc001728a00?, 0x7f51f42af700?, 0x7f523c789f30?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc001105910 sp=0xc0011058c8 pc=0x3183385\ncrypto/tls.(*atLeastReader).Read(0xc001800f60, {0xc001728a00?, 0x240a?, 0x0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc001105958 sp=0xc001105910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000c129a8, {0x9220320, 0xc001800f60})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0011059b0 sp=0xc001105958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000c12708, {0x9221e00, 0xc000d6aa10}, 0xc001105a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0011059e8 sp=0xc0011059b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c12708, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc001105c60 sp=0xc0011059e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c12708, {0xc001892000, 0x1000, 0xc0008673b0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc001105cc8 sp=0xc001105c60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc0018445a0, {0xc001892000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc001105d28 sp=0xc001105cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc000e9db60)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc001105d60 sp=0xc001105d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc000e9db60, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc001105d80 sp=0xc001105d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0018445a0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc001105fc8 sp=0xc001105d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc001105fe0 sp=0xc001105fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001105fe8 sp=0xc001105fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 197\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax    0xca\nrbx    0x0\nrcx    0x2f28ae3\nrdx    0x0\nrdi    0xc90cc70\nrsi    0x80\nrbp    0x7fff4aa32180\nrsp    0x7fff4aa32138\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x7fff4aa32180\nr13    0x1\nr14    0xc90a6e0\nr15    0x5\nrip    0x2f28ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n53 fp=0xc0000c9d80 sp=0xc0000c9d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0010a6b40)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0000c9fc8 sp=0xc0000c9d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0000c9fe0 sp=0xc0000c9fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c9fe8 sp=0xc0000c9fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 212\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 236 gp=0xc000e1ec40 m=nil [select, 2 minutes]:\nruntime.gopark(0xc001303f48?, 0x2?, 0xc0?, 0x3d?, 0xc001303ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000382d78 sp=0xc000382d58 pc=0x2f1e50e\nruntime.selectgo(0xc000382f48, 0xc001303ef0, 0xc0008e8b40?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000382eb8 sp=0xc000382d78 pc=0x2ef92f7\nnet/http.(*persistConn).writeLoop(0xc0018445a0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000382fc8 sp=0xc000382eb8 pc=0x32e6ba7\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000382fe0 sp=0xc000382fc8 pc=0x32e3845\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000382fe8 sp=0xc000382fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 197\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 235 gp=0xc000682e00 m=nil [IO wait, 2 minutes]:\nruntime.gopark(0xc0000d5760?, 0xc001105788?, 0xc5?, 0xbe?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001105768 sp=0xc001105748 pc=0x2f1e50e\nruntime.netpollblock(0x2f88ad8?, 0x2eaa066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0011057a0 sp=0xc001105768 pc=0x2eddfb7\ninternal/poll.runtime_pollWait(0x7f51f5ba3e00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0011057c0 sp=0xc0011057a0 pc=0x2f1cda5\ninternal/poll.(*pollDesc).wait(0xc001866680?, 0xc001728a00?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0011057e8 sp=0xc0011057c0 pc=0x2fab907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc001866680, {0xc001728a00, 0x2500, 0x2500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc001105880 sp=0xc0011057e8 pc=0x2facd39\nnet.(*netFD).Read(0xc001866680, {0xc001728a00?, 0xc001728a00?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0011058c8 sp=0xc001105880 pc=0x31712e5\nnet.(*conn).Read(0xc000d6aa10, {0xc001728a00?, 0x7f51f42af700?, 0x7f523c789f30?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc001105910 sp=0xc0011058c8 pc=0x3183385\ncrypto/tls.(*atLeastReader).Read(0xc001800f60, {0xc001728a00?, 0x240a?, 0x0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc001105958 sp=0xc001105910 pc=0x31eef1b\nbytes.(*Buffer).ReadFrom(0xc000c129a8, {0x9220320, 0xc001800f60})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0011059b0 sp=0xc001105958 pc=0x2fef578\ncrypto/tls.(*Conn).readFromUntil(0xc000c12708, {0x9221e00, 0xc000d6aa10}, 0xc001105a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0011059e8 sp=0xc0011059b0 pc=0x31ef0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c12708, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc001105c60 sp=0xc0011059e8 pc=0x31ec17b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c12708, {0xc001892000, 0x1000, 0xc0008673b0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc001105cc8 sp=0xc001105c60 pc=0x31f2a85\nnet/http.(*persistConn).Read(0xc0018445a0, {0xc001892000?, 0x32dcb05?, 0x921acc0?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc001105d28 sp=0xc001105cc8 pc=0x32e4407\nbufio.(*Reader).fill(0xc000e9db60)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc001105d60 sp=0xc001105d28 pc=0x3084943\nbufio.(*Reader).Peek(0xc000e9db60, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc001105d80 sp=0xc001105d60 pc=0x3084ad3\nnet/http.(*persistConn).readLoop(0xc0018445a0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc001105fc8 sp=0xc001105d80 pc=0x32e4f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc001105fe0 sp=0xc001105fc8 pc=0x32e38a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001105fe8 sp=0xc001105fe0 pc=0x2f26cc1\ncreated by net/http.(*Transport).dialConn in goroutine 197\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax    0xca\nrbx    0x0\nrcx    0x2f28ae3\nrdx    0x0\nrdi    0xc90cc70\nrsi    0x80\nrbp    0x7fff4aa32180\nrsp    0x7fff4aa32138\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x7fff4aa32180\nr13    0x1\nr14    0xc90a6e0\nr15    0x5\nrip    0x2f28ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","178c2b99062d6c6ed58013715ed872d5fb2a585b82154be38dbf861ac0377659","1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","314b58d43728f942111a52209bcc085b0bf14adcab05fa6fc6959abcb89c83c4","7816b6118b6d30763d9aeef2698a47954586af9058cb4051aa7efd88e0110aab","78754e29456c1fc88cbae908323785a403c4346735c925fc3102d3c04ce7e5df","a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068399947908124672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T18:25:51Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"877121df16c97517ecd55b1c9ec4a8a0da2bb6dc34b32528a0d7cd3698e555ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068399947908124672","occurred_at":"2026-06-20T18:25:51Z","failed_at":"e2e","signature_id":"bdb0464d6ce770897024e62edf4ebdb3e733526a04930e143fcc58837d12f7c9","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00106a780\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-26pckg, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26pckg\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00106a780\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-26pckg, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26pckg\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068274817072631808","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-20T10:08:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068274747245858816","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-20T10:08:21Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068274664437714944","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T10:08:01Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"41f7f1f427a3018dae3b6dc9b560cd2a71f5d00ff5c03e2999674accf063bff0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068274664437714944","occurred_at":"2026-06-20T10:08:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068216057461477376","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T06:15:08Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6745a0337e21f92c6cad81dd42831c7209282aafb191c8ee79b71453e9e5215f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068216057461477376","occurred_at":"2026-06-20T06:15:08Z","failed_at":"e2e","signature_id":"1e2e5dcf69c1b15b0879893ecc53787a77e157be5eb9f8f914bdb5e424da68aa","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-qmf5n6\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007ccbd0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-qmf5n6, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-qmf5n6 in resource group rg-zstream-upgrade-4-22-swxbzb, error: failed waiting for cluster=\"cluster-zstream-4-22-qmf5n6\" in resourcegroup=\"rg-zstream-upgrade-4-22-swxbzb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-qmf5n6 in resource group rg-zstream-upgrade-4-22-swxbzb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-qmf5n6\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007ccbd0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-qmf5n6, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-qmf5n6 in resource group rg-zstream-upgrade-4-22-swxbzb, error: failed waiting for cluster=\"cluster-zstream-4-22-qmf5n6\" in resourcegroup=\"rg-zstream-upgrade-4-22-swxbzb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-qmf5n6 in resource group rg-zstream-upgrade-4-22-swxbzb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068216024569745408","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T06:15:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3f8ccf37b88b5a6906f5fd89b0409ae8eb88f64f5b2eee51a23ca9ea051c3a34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068216024569745408","occurred_at":"2026-06-20T06:15:00Z","failed_at":"e2e","signature_id":"e404ee63f95099b59271a15602d45742ef4fc421991f9967da4ec935bd09ed2c","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007bf3b0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:132]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007bf3b0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068215937068175360","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-20T06:14:39Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"97e6402bb5d42076523d96168dd363a09e4dd342ba81bf39b362199325326149","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2068215937068175360","occurred_at":"2026-06-20T06:14:39Z","failed_at":"e2e","signature_id":"3a75c96a10ecf612dd2f7d814a539cc03f1bbd30ad0eca88d410a8c39cb508a3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-dmjkpc\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009ef110\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dmjkpc, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-dmjkpc in resource group rg-zstream-upgrade-4-22-jzf2r6, error: failed waiting for cluster=\"cluster-zstream-4-22-dmjkpc\" in resourcegroup=\"rg-zstream-upgrade-4-22-jzf2r6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-dmjkpc in resource group rg-zstream-upgrade-4-22-jzf2r6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-dmjkpc\" with version 4.22.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009ef110\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-dmjkpc, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-dmjkpc in resource group rg-zstream-upgrade-4-22-jzf2r6, error: failed waiting for cluster=\"cluster-zstream-4-22-dmjkpc\" in resourcegroup=\"rg-zstream-upgrade-4-22-jzf2r6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-dmjkpc in resource group rg-zstream-upgrade-4-22-jzf2r6, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-f4gmfq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: failed waiting for cluster=\"cluster-candidate-4-22-f4gmfq\" in resourcegroup=\"rg-candidate-4-22-lklm69\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-f4gmfq in resource group rg-candidate-4-22-lklm69, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2068151960195829760","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-20T02:00:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}