{"meta":{"date":"2026-04-24","timezone":"UTC","generated_at":"2026-08-30T06:17:17Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-24T18:17:55Z"},"failed_at":["e2e"],"failed_test_count":29,"occurrences":[{"row_id":"05011004b48081a4045b7577edd25b0c3ed09440e21ee23a55d1e6faf68724be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal"}},{"row_id":"06a56b85f52b5c747a8004b37405d27d2528d08cb701fa2f33d865b9137e5ad8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"aabfb8cea29a0d179fe6ec205e918b3b7ae4ae354c1a30b9d4b895d189e11003","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00133a780\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-s55xz8, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-s55xz8 in resource group rg-zstream-upgrade-4-21-mnx284, error: failed waiting for cluster=\"cluster-zstream-4-21-s55xz8\" in resourcegroup=\"rg-zstream-upgrade-4-21-mnx284\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-s55xz8 in resource group rg-zstream-upgrade-4-21-mnx284, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00133a780\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-s55xz8, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-s55xz8 in resource group rg-zstream-upgrade-4-21-mnx284, error: failed waiting for cluster=\"cluster-zstream-4-21-s55xz8\" in resourcegroup=\"rg-zstream-upgrade-4-21-mnx284\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-s55xz8 in resource group rg-zstream-upgrade-4-21-mnx284, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"089ef6eafc118751d9f71bc5ce61837fc3c0144ae979bc984e3be1dd268ca80a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"b06d6a76c2852dfffc3f3a52892611ed51c1ee0df7e6f5841a7a83fdefa28487","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00100bc50\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-nwngq9, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-nwngq9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-nwngq9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00100bc50\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-nwngq9, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-nwngq9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-nwngq9, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"09948ab388a18315ae655bbbdbe070bed065b02c98d523d425c71280aac6d962","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"7ea808d7a2819dfbbb0955a17c09815dfdeabfda6098193eb2435310de4389d7","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cfd4d0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-20-lmh9qb\" in resourcegroup=\"rg-zstream-upgrade-4-20-pjcrzv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-lmh9qb in resource group rg-zstream-upgrade-4-20-pjcrzv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cfd4d0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-20-lmh9qb\" in resourcegroup=\"rg-zstream-upgrade-4-20-pjcrzv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-lmh9qb in resource group rg-zstream-upgrade-4-20-pjcrzv, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-zstream-4-20-lmh9qb\" in resourcegroup=\"rg-zstream-upgrade-4-20-pjcrzv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-lmh9qb in resource group rg-zstream-upgrade-4-20-pjcrzv, error: context deadline exceeded"}},{"row_id":"0e1a8a84893f2336432284b1b4b050a6d1f74e705cf69736e677875011323c49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"c3f3b3c04d0f9d55622e585be63d3577b725c1576b4458f258351592d55fc08f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009be750\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-9cnx7f, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9cnx7f in resource group rg-np-version-upgrade-9cnx7f-crs5mk, error: failed waiting for cluster=\"np-version-upgrade-cluster-9cnx7f\" in resourcegroup=\"rg-np-version-upgrade-9cnx7f-crs5mk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9cnx7f in resource group rg-np-version-upgrade-9cnx7f-crs5mk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009be750\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-9cnx7f, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9cnx7f in resource group rg-np-version-upgrade-9cnx7f-crs5mk, error: failed waiting for cluster=\"np-version-upgrade-cluster-9cnx7f\" in resourcegroup=\"rg-np-version-upgrade-9cnx7f-crs5mk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9cnx7f in resource group rg-np-version-upgrade-9cnx7f-crs5mk, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"1646e85aaa985b861d77df156bfa79b2b19c9d0a3d97af692cded6f4267cd35f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"81a038536c4312c97a10ccb7cbcc503a577979e412955ce6733518b54b4004d0","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009d9500\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-cvqx4g, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-cvqx4g\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009d9500\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-cvqx4g, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-cvqx4g\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kpr8f5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kpr8f5\" to finish creating: context deadline exceeded"}},{"row_id":"178cc596ae3b5bad1d6026a6101ddbe4bfdd508346b335ae1256c5500ffd51fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"b3ce37421496a578a4b9a787cad5d606f4657435b3f432b26f60d042bc991f11","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:256]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f602d0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-s5wf8z, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-s5wf8z\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-s5wf8z, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:256]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f602d0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-s5wf8z, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-s5wf8z\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-s5wf8z, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"2f738f078975802e59abb0a0654ce9fbd00421dcbc834fafe822022ee88e8a97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"90e4e34401463acba6c3573ab41dffad0f272b7b03e5b5baa92d2cb894b15de9","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00258ae10\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-h2vkbm, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h2vkbm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-h2vkbm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00258ae10\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-h2vkbm, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h2vkbm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-h2vkbm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"2f8ab7dbce69e60a024e9ada4cbbc99f22e282b5a9a6bc7ac9c689b8942c4048","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"cb205895af951d198f72cc650fdb162c41a56581cf216a81b8b17d8ec1af2496","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001572f00\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5jjsrz, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5jjsrz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5jjsrz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001572f00\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5jjsrz, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5jjsrz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5jjsrz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"4b5a3d2a0c6cea16e191d3392722fea51df84db71e727d26f6d530dbdcfd918a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"15370f7770de299dd21fa60d61b5548f9cade8895d265d537cc66db2f69caf2e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a671d0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m9xxfm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-m9xxfm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a671d0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-m9xxfm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-m9xxfm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"6358bfeb4b2f661b8ce90bd0b0134d44a82c77daac6a49e8fed338fd2af7bed4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"d2b32b3e6183f632a51d43179d4b9ccd70b9da37b667597cbc1014d0f19322f6","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00167e480\u003e: \n    failed to create HCP cluster ea-listtlh8kj, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtlh8kj in resource group ea-list-84xzfb, error: failed waiting for cluster=\"ea-listtlh8kj\" in resourcegroup=\"ea-list-84xzfb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtlh8kj in resource group ea-list-84xzfb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00167e480\u003e: \n    failed to create HCP cluster ea-listtlh8kj, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtlh8kj in resource group ea-list-84xzfb, error: failed waiting for cluster=\"ea-listtlh8kj\" in resourcegroup=\"ea-list-84xzfb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listtlh8kj in resource group ea-list-84xzfb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"668ef01328b9ec7b0eee8eb9571f7e45687f0e6ffe21e58f652ac47988c267ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"9beab1ecfa673ce13d1f36103123024a9be671eae0104f6d17b921a99c84f37f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f44750\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5m44zw, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-21-5m44zw in resource group rg-candidate-4-21-zxwhzl, error: failed waiting for cluster=\"cluster-candidate-4-21-5m44zw\" in resourcegroup=\"rg-candidate-4-21-zxwhzl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-21-5m44zw in resource group rg-candidate-4-21-zxwhzl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f44750\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5m44zw, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-21-5m44zw in resource group rg-candidate-4-21-zxwhzl, error: failed waiting for cluster=\"cluster-candidate-4-21-5m44zw\" in resourcegroup=\"rg-candidate-4-21-zxwhzl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-21-5m44zw in resource group rg-candidate-4-21-zxwhzl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"6867a83621588c4a24e14849e03060bc83331af56aec183c16f9dfa5298fc69d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"9bf0b88b14f49ca8939e47c5f49f7451c2fe89b52d6654d0baaf34e317690c40","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000907200\u003e: \n    failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9b6btl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-9b6btl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000907200\u003e: \n    failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9b6btl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-9b6btl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-zstream-4-20-lmh9qb\" in resourcegroup=\"rg-zstream-upgrade-4-20-pjcrzv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-lmh9qb in resource group rg-zstream-upgrade-4-20-pjcrzv, error: context deadline exceeded"}},{"row_id":"7da3a08c3fc3d3d65877dbd2b5a81f0984488709952dd0ee324e0476e174fd7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"cf9d93b210380046fcc5f0bb98b4252e493abe88d50568fbb25ef6cdb25b4835","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00138c840\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-8jdpgv, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8jdpgv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-8jdpgv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00138c840\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-8jdpgv, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8jdpgv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-8jdpgv, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"8e2a621c58e73ebab28d71da58406633a7e6d6f98a388b7bf533322066cb7981","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"31d893f41adc5c1cd26e71d1848f6a631e9135202d3c807638ec907e2acca29f","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009ee5d0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009ee5d0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"8fe0526f52bf685a69c4623814cde2cb96a40eaea2fb1a5632f3f4dad49b0570","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"8cecea8915e316fe9076db2275bc1d6b45f4ade5b0c59dcb9235d6dbd7f98c1d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"b78 pc=0x7addf0e\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000e97fe0 sp=0xc000e97c20 pc=0x7ad9c76\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e97fe8 sp=0xc000e97fe0 pc=0x34fbcc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 154 gp=0xc0006036c0 m=nil [IO wait]:\nruntime.gopark(0xc000c94a20?, 0x84f4a900?, 0x0?, 0xd6?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e1d720 sp=0xc000e1d700 pc=0x34f350e\nruntime.netpollblock(0x355dd18?, 0x347f066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e1d758 sp=0xc000e1d720 pc=0x34b2fb7\ninternal/poll.runtime_pollWait(0x7fd885f9b000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000e1d778 sp=0xc000e1d758 pc=0x34f1da5\ninternal/poll.(*pollDesc).wait(0xc0005dcc80?, 0xc000e24000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e1d7a0 sp=0xc000e1d778 pc=0x3580a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc0005dcc80, {0xc000e24000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e1d838 sp=0xc000e1d7a0 pc=0x3581e79\nnet.(*netFD).Read(0xc0005dcc80, {0xc000e24000?, 0xc000e242f2?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000e1d880 sp=0xc000e1d838 pc=0x37475e5\nnet.(*conn).Read(0xc000d96418, {0xc000e24000?, 0x7fd8844dfd00?, 0x7fd8cdbc3a78?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000e1d8c8 sp=0xc000e1d880 pc=0x3759685\ncrypto/tls.(*atLeastReader).Read(0xc000ada018, {0xc000e24000?, 0x3209?, 0x34f0245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e1d910 sp=0xc000e1d8c8 pc=0x37c511b\nbytes.(*Buffer).ReadFrom(0xc000c717a8, {0x9ff3c00, 0xc000ada018})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000e1d968 sp=0xc000e1d910 pc=0x35bec78\ncrypto/tls.(*Conn).readFromUntil(0xc000c71508, {0x9ff47a0, 0xc000d96418}, 0xc000e1d9d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000e1d9a0 sp=0xc000e1d968 pc=0x37c52fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c71508, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e1dc18 sp=0xc000e1d9a0 pc=0x37c237b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c71508, {0xc00131d000, 0x1000, 0x3bc1820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e1dc80 sp=0xc000e1dc18 pc=0x37c8c65\nbufio.(*Reader).Read(0xc0012fa6c0, {0xc000156c84, 0x9, 0x3bcf8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000e1dcb8 sp=0xc000e1dc80 pc=0x365b1d7\nio.ReadAtLeast({0x9ff3200, 0xc0012fa6c0}, {0xc000156c84, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000e1dd00 sp=0xc000e1dcb8 pc=0x354de8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000156c84, 0x9, 0xc000000211?}, {0x9ff3200?, 0xc0012fa6c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000e1dd50 sp=0xc000e1dd00 pc=0x3bb85e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000156c40)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000e1ddd8 sp=0xc000e1dd50 pc=0x3bb91ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000156c40)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000e1de18 sp=0xc000e1ddd8 pc=0x3bb9918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e1dfa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000e1df60 sp=0xc000e1de18 pc=0x3bcc7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc00140a8c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000e1dfc8 sp=0xc000e1df60 pc=0x3bcbb72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000e1dfe0 sp=0xc000e1dfc8 pc=0x3bc5305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e1dfe8 sp=0xc000e1dfe0 pc=0x34fbcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 153\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x34fdae3\nrdx    0x0\nrdi    0xdba8e10\nrsi    0x80\nrbp    0x7fd886fcfcb8\nrsp    0x7fd886fcfc70\nr8     0x0\nr9     0x0\nr10    0x7fd886fcfca8\nr11    0x202\nr12    0x7fd886fcfca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x34fdae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nb78 pc=0x7addf0e\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000e97fe0 sp=0xc000e97c20 pc=0x7ad9c76\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e97fe8 sp=0xc000e97fe0 pc=0x34fbcc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 154 gp=0xc0006036c0 m=nil [IO wait]:\nruntime.gopark(0xc000c94a20?, 0x84f4a900?, 0x0?, 0xd6?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e1d720 sp=0xc000e1d700 pc=0x34f350e\nruntime.netpollblock(0x355dd18?, 0x347f066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e1d758 sp=0xc000e1d720 pc=0x34b2fb7\ninternal/poll.runtime_pollWait(0x7fd885f9b000, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000e1d778 sp=0xc000e1d758 pc=0x34f1da5\ninternal/poll.(*pollDesc).wait(0xc0005dcc80?, 0xc000e24000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e1d7a0 sp=0xc000e1d778 pc=0x3580a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc0005dcc80, {0xc000e24000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e1d838 sp=0xc000e1d7a0 pc=0x3581e79\nnet.(*netFD).Read(0xc0005dcc80, {0xc000e24000?, 0xc000e242f2?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000e1d880 sp=0xc000e1d838 pc=0x37475e5\nnet.(*conn).Read(0xc000d96418, {0xc000e24000?, 0x7fd8844dfd00?, 0x7fd8cdbc3a78?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000e1d8c8 sp=0xc000e1d880 pc=0x3759685\ncrypto/tls.(*atLeastReader).Read(0xc000ada018, {0xc000e24000?, 0x3209?, 0x34f0245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e1d910 sp=0xc000e1d8c8 pc=0x37c511b\nbytes.(*Buffer).ReadFrom(0xc000c717a8, {0x9ff3c00, 0xc000ada018})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000e1d968 sp=0xc000e1d910 pc=0x35bec78\ncrypto/tls.(*Conn).readFromUntil(0xc000c71508, {0x9ff47a0, 0xc000d96418}, 0xc000e1d9d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000e1d9a0 sp=0xc000e1d968 pc=0x37c52fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000c71508, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e1dc18 sp=0xc000e1d9a0 pc=0x37c237b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000c71508, {0xc00131d000, 0x1000, 0x3bc1820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e1dc80 sp=0xc000e1dc18 pc=0x37c8c65\nbufio.(*Reader).Read(0xc0012fa6c0, {0xc000156c84, 0x9, 0x3bcf8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000e1dcb8 sp=0xc000e1dc80 pc=0x365b1d7\nio.ReadAtLeast({0x9ff3200, 0xc0012fa6c0}, {0xc000156c84, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000e1dd00 sp=0xc000e1dcb8 pc=0x354de8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc000156c84, 0x9, 0xc000000211?}, {0x9ff3200?, 0xc0012fa6c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000e1dd50 sp=0xc000e1dd00 pc=0x3bb85e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc000156c40)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000e1ddd8 sp=0xc000e1dd50 pc=0x3bb91ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc000156c40)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000e1de18 sp=0xc000e1ddd8 pc=0x3bb9918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e1dfa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000e1df60 sp=0xc000e1de18 pc=0x3bcc7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc00140a8c0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000e1dfc8 sp=0xc000e1df60 pc=0x3bcbb72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000e1dfe0 sp=0xc000e1dfc8 pc=0x3bc5305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e1dfe8 sp=0xc000e1dfe0 pc=0x34fbcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 153\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x34fdae3\nrdx    0x0\nrdi    0xdba8e10\nrsi    0x80\nrbp    0x7fd886fcfcb8\nrsp    0x7fd886fcfc70\nr8     0x0\nr9     0x0\nr10    0x7fd886fcfca8\nr11    0x202\nr12    0x7fd886fcfca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x34fdae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"92fbab2b65a4b94117b2305523782fbe53f1442a1ddec1c7a34bad715a1b3352","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"d8adf81bb8ddcd49b4d64de9b008b395d4ac7042097b5453fa3f5cf731114b29","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001722810\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dxfxqf, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-dxfxqf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dxfxqf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001722810\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dxfxqf, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-dxfxqf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-dxfxqf, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"9a2da40e697261fb798522bc6a99d8e13fe761830c55e429bcca90f968a80182","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"d5331347e2faf79329c3cdafc6a44e8ec7df1e1af045fc0133e9a7f3146e64a0","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d74480\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-xs4bxm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d74480\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-xs4bxm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-xs4bxm, error: context deadline exceeded"}},{"row_id":"9dbd7bf58b4d9eadf53b1cea473bfd9bafb22a73629209e6a80eecb20c3e4eb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"c1a77945ba81ac7509d545df31cfac708dc285cf96391246d2a02928ddb6d90d","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a5a780\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-xsgg2.apps.aro.b7c9d0c5o8o2d7a.ldkt.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a5a780\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-xsgg2.apps.aro.b7c9d0c5o8o2d7a.ldkt.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"a0af2ddcc70a4c3ce0ab1a4697b1bd549d3a469b249311b6997a96511e9be0f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"d9dc6aa889ea69ad197f41ed2afe30f8cfc2a7d6da44938a3e4e287480e4c19e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:250] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000013098\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9fbgg8\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:250] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000013098\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9fbgg8\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9fbgg8\" to finish deleting: context canceled"}},{"row_id":"a32ae72a7b33ec91a6901e396c17e5d17af511411ffb6ef01f58966b00cc0bc4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"83c702468523619fbe6379c16bc2a83ff8db23f70da8474d809616c96fb20f7a","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000416240\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-8vt699, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8vt699\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-8vt699, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000416240\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-8vt699, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8vt699\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-8vt699, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"ae7f64b9bff44710adb774594d41fc5bfd64fcce8cdb689e80229246df438c6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"67a15279bed7445a314209be249de1dd78609844878114eb517f01012354f7bd","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e607e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-csg2sf, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-csg2sf in resource group rg-np-version-upgrade-csg2sf-dmkhjr, error: failed waiting for cluster=\"np-version-upgrade-cluster-csg2sf\" in resourcegroup=\"rg-np-version-upgrade-csg2sf-dmkhjr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-csg2sf in resource group rg-np-version-upgrade-csg2sf-dmkhjr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e607e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-csg2sf, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-csg2sf in resource group rg-np-version-upgrade-csg2sf-dmkhjr, error: failed waiting for cluster=\"np-version-upgrade-cluster-csg2sf\" in resourcegroup=\"rg-np-version-upgrade-csg2sf-dmkhjr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-csg2sf in resource group rg-np-version-upgrade-csg2sf-dmkhjr, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"b1c28757ec92d7a48f4b721e4dc9f4eeefc68e499d013ef067eda5cc5172808d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"80558011f1c9e50f3e5aa79df8c7afbf4eb98cf5863b74baa0224d0ee6e5c781","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015ec570\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-xlm4j8, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-xlm4j8 in resource group rg-cp-ystream-upgrade-4-21-7h74xf, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xlm4j8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-7h74xf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-xlm4j8 in resource group rg-cp-ystream-upgrade-4-21-7h74xf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015ec570\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-xlm4j8, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-xlm4j8 in resource group rg-cp-ystream-upgrade-4-21-7h74xf, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xlm4j8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-7h74xf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-xlm4j8 in resource group rg-cp-ystream-upgrade-4-21-7h74xf, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"ccc14321b4cc461681b8ead28458176f5185b1be1f5c512cad6748bc130134fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"aa6d5a70477f4615dfac5aab96b8f8a7cbd905a70827fd3203948ca242c3fddc","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014f84b0\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-l9rjzt, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-l9rjzt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-l9rjzt, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014f84b0\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-l9rjzt, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-l9rjzt\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-l9rjzt, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"d17c90b4f2f34f659c898eed0ba542a3997466d3a11992b1cb4d69ccfdc1ddd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"1f7351eb1f3c9c713230b3d6aef5de456ff9923a7313aa4ce0582e85fb645aac","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016622a0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-lhx428, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-lhx428 in resource group rg-np-version-upgrade-lhx428-28c89f, error: failed waiting for cluster=\"np-version-upgrade-cluster-lhx428\" in resourcegroup=\"rg-np-version-upgrade-lhx428-28c89f\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-lhx428 in resource group rg-np-version-upgrade-lhx428-28c89f, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016622a0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-lhx428, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-lhx428 in resource group rg-np-version-upgrade-lhx428-28c89f, error: failed waiting for cluster=\"np-version-upgrade-cluster-lhx428\" in resourcegroup=\"rg-np-version-upgrade-lhx428-28c89f\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-lhx428 in resource group rg-np-version-upgrade-lhx428-28c89f, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"d705aacd12c5d9337f650be035e5d05adf3b8fe4e5a40e54cef1463e338af779","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"f16cff6c452324b1a7add76b103260b56495045e2dca2aed19839af0d7d5eebc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"od/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000dc4b88, {0xc000d76c20?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc00090c008, {0x0, 0x0}, {0xdbd1a20, 0x0, 0x0}, {0xc0000be094, 0x2a}, 0x0, {0xc000d76c00, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000be094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x3482a85?, 0xc000379da0?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0xa0b59c8, 0xc000a93770}, 0xc0009a8d00, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 101 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc0009a8d00?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 100\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115 @ 04/24/26 20:48:26.049\n  \"ts\"=\"2026-04-24 20:48:26.050189\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-24 20:48:26.050494\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-s7rbn7-snjkqn\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\"\n  \"ts\"=\"2026-04-24 20:48:26.240300\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-20-s7rbn7-snjkqn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-24 20:48:28.478070\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-04-24 20:48:28.478235\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-04-24 20:48:28.478350\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-24 20:48:28.481215\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-35\"\n  \"ts\"=\"2026-04-24 20:48:28.481257\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\"\n  \"ts\"=\"2026-04-24 20:48:28.638712\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-35\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-24 20:48:28.656689\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\" \"deployment\"=\"pooledNonMsiScopedAssignments-yrgv7gacpfvfw\"\n  \"ts\"=\"2026-04-24 20:48:28.793120\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-s7rbn7-snjkqn\"\n  \"ts\"=\"2026-04-24 20:48:28.804040\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-35\" \"deployment\"=\"msi-deployment-35\"\n  \"ts\"=\"2026-04-24 20:48:28.956124\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-24 20:48:29.354732\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-24 20:48:29.354794\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000dc4b88, {0xc000d76c20?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc00090c008, {0x0, 0x0}, {0xdbd1a20, 0x0, 0x0}, {0xc0000be094, 0x2a}, 0x0, {0xc000d76c00, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc0000be094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x3482a85?, 0xc000379da0?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0xa0b59c8, 0xc000a93770}, 0xc0009a8d00, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 101 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc0009a8d00?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 100\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115 @ 04/24/26 20:48:26.049\n  \"ts\"=\"2026-04-24 20:48:26.050189\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-24 20:48:26.050494\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-s7rbn7-snjkqn\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\"\n  \"ts\"=\"2026-04-24 20:48:26.240300\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-20-s7rbn7-snjkqn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-24 20:48:28.478070\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-04-24 20:48:28.478235\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-04-24 20:48:28.478350\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-24 20:48:28.481215\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-35\"\n  \"ts\"=\"2026-04-24 20:48:28.481257\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\"\n  \"ts\"=\"2026-04-24 20:48:28.638712\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-35\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-24 20:48:28.656689\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\" \"deployment\"=\"pooledNonMsiScopedAssignments-yrgv7gacpfvfw\"\n  \"ts\"=\"2026-04-24 20:48:28.793120\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-s7rbn7-snjkqn\"\n  \"ts\"=\"2026-04-24 20:48:28.804040\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-35\" \"deployment\"=\"msi-deployment-35\"\n  \"ts\"=\"2026-04-24 20:48:28.956124\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-24 20:48:29.354732\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-24 20:48:29.354794\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-20-hnr4ms\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e6227c65c8b1d892b8b4982a734de9d0e858a3f3dfbf5f0233ec5620a1dbf11f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"70dd085f4a557e8a91bf2c6b80f80b59a30dd84ecebc294e6ac8a55a2d45aeba","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfaf90\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-46g97d, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-46g97d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-46g97d, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfaf90\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-46g97d, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-46g97d\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-46g97d, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"e822fa6285e6b44e4757de44a16dc0de7b92dd6d840def9e7ff5d7d0b16d3e10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"dcaaecc2f5339a1c97f554317e0c4140b84b4db3579805e70f8a471d03dc8303","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"thub.com/onsi/ginkgo/v2.generateTable.func2.1({0xa0d19a0?, 0xc00090e240?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc0006b5f88 sp=0xc0006b5f18 pc=0x3979245\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc0006b5fe0 sp=0xc0006b5f88 pc=0x396abbb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0006b5fe8 sp=0xc0006b5fe0 pc=0x34fbcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 96\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 166 gp=0xc000c0b6c0 m=nil [IO wait]:\nruntime.gopark(0xc000a218e0?, 0x0?, 0x0?, 0xf?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000131720 sp=0xc000131700 pc=0x34f350e\nruntime.netpollblock(0x355dd18?, 0x347f066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000131758 sp=0xc000131720 pc=0x34b2fb7\ninternal/poll.runtime_pollWait(0x7fa1cac41600, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000131778 sp=0xc000131758 pc=0x34f1da5\ninternal/poll.(*pollDesc).wait(0xc000747c80?, 0xc0015e0000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0001317a0 sp=0xc000131778 pc=0x3580a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000747c80, {0xc0015e0000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000131838 sp=0xc0001317a0 pc=0x3581e79\nnet.(*netFD).Read(0xc000747c80, {0xc0015e0000?, 0xc0015e02ca?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000131880 sp=0xc000131838 pc=0x37475e5\nnet.(*conn).Read(0xc000f802b0, {0xc0015e0000?, 0x7fa1c8a7e0f0?, 0x7fa21306a5c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc0001318c8 sp=0xc000131880 pc=0x3759685\ncrypto/tls.(*atLeastReader).Read(0xc000bec468, {0xc0015e0000?, 0x3231?, 0x34f0245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000131910 sp=0xc0001318c8 pc=0x37c511b\nbytes.(*Buffer).ReadFrom(0xc0005c5ea8, {0x9ff3c00, 0xc000bec468})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000131968 sp=0xc000131910 pc=0x35bec78\ncrypto/tls.(*Conn).readFromUntil(0xc0005c5c08, {0x9ff47a0, 0xc000f802b0}, 0xc0001319d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0001319a0 sp=0xc000131968 pc=0x37c52fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0005c5c08, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000131c18 sp=0xc0001319a0 pc=0x37c237b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0005c5c08, {0xc001186000, 0x1000, 0x3bc1820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000131c80 sp=0xc000131c18 pc=0x37c8c65\nbufio.(*Reader).Read(0xc000ca55c0, {0xc001012744, 0x9, 0x3bcf8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000131cb8 sp=0xc000131c80 pc=0x365b1d7\nio.ReadAtLeast({0x9ff3200, 0xc000ca55c0}, {0xc001012744, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000131d00 sp=0xc000131cb8 pc=0x354de8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc001012744, 0x9, 0xc00000012d?}, {0x9ff3200?, 0xc000ca55c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000131d50 sp=0xc000131d00 pc=0x3bb85e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc001012700)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000131dd8 sp=0xc000131d50 pc=0x3bb91ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc001012700)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000131e18 sp=0xc000131dd8 pc=0x3bb9918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000131fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000131f60 sp=0xc000131e18 pc=0x3bcc7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000603880)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000131fc8 sp=0xc000131f60 pc=0x3bcbb72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000131fe0 sp=0xc000131fc8 pc=0x3bc5305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000131fe8 sp=0xc000131fe0 pc=0x34fbcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 165\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x34fdae3\nrdx    0x0\nrdi    0xdba6cd0\nrsi    0x80\nrbp    0x7ffe490bad98\nrsp    0x7ffe490bad50\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x5\nr13    0x1\nr14    0xdba4be0\nr15    0x5\nrip    0x34fdae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nthub.com/onsi/ginkgo/v2.generateTable.func2.1({0xa0d19a0?, 0xc00090e240?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc0006b5f88 sp=0xc0006b5f18 pc=0x3979245\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc0006b5fe0 sp=0xc0006b5f88 pc=0x396abbb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0006b5fe8 sp=0xc0006b5fe0 pc=0x34fbcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 96\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 166 gp=0xc000c0b6c0 m=nil [IO wait]:\nruntime.gopark(0xc000a218e0?, 0x0?, 0x0?, 0xf?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000131720 sp=0xc000131700 pc=0x34f350e\nruntime.netpollblock(0x355dd18?, 0x347f066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000131758 sp=0xc000131720 pc=0x34b2fb7\ninternal/poll.runtime_pollWait(0x7fa1cac41600, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000131778 sp=0xc000131758 pc=0x34f1da5\ninternal/poll.(*pollDesc).wait(0xc000747c80?, 0xc0015e0000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0001317a0 sp=0xc000131778 pc=0x3580a47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000747c80, {0xc0015e0000, 0x3500, 0x3500})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000131838 sp=0xc0001317a0 pc=0x3581e79\nnet.(*netFD).Read(0xc000747c80, {0xc0015e0000?, 0xc0015e02ca?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000131880 sp=0xc000131838 pc=0x37475e5\nnet.(*conn).Read(0xc000f802b0, {0xc0015e0000?, 0x7fa1c8a7e0f0?, 0x7fa21306a5c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc0001318c8 sp=0xc000131880 pc=0x3759685\ncrypto/tls.(*atLeastReader).Read(0xc000bec468, {0xc0015e0000?, 0x3231?, 0x34f0245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000131910 sp=0xc0001318c8 pc=0x37c511b\nbytes.(*Buffer).ReadFrom(0xc0005c5ea8, {0x9ff3c00, 0xc000bec468})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000131968 sp=0xc000131910 pc=0x35bec78\ncrypto/tls.(*Conn).readFromUntil(0xc0005c5c08, {0x9ff47a0, 0xc000f802b0}, 0xc0001319d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0001319a0 sp=0xc000131968 pc=0x37c52fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0005c5c08, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000131c18 sp=0xc0001319a0 pc=0x37c237b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0005c5c08, {0xc001186000, 0x1000, 0x3bc1820?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000131c80 sp=0xc000131c18 pc=0x37c8c65\nbufio.(*Reader).Read(0xc000ca55c0, {0xc001012744, 0x9, 0x3bcf8ae?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000131cb8 sp=0xc000131c80 pc=0x365b1d7\nio.ReadAtLeast({0x9ff3200, 0xc000ca55c0}, {0xc001012744, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000131d00 sp=0xc000131cb8 pc=0x354de8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc001012744, 0x9, 0xc00000012d?}, {0x9ff3200?, 0xc000ca55c0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000131d50 sp=0xc000131d00 pc=0x3bb85e5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc001012700)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000131dd8 sp=0xc000131d50 pc=0x3bb91ab\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc001012700)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000131e18 sp=0xc000131dd8 pc=0x3bb9918\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000131fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000131f60 sp=0xc000131e18 pc=0x3bcc7ea\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000603880)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000131fc8 sp=0xc000131f60 pc=0x3bcbb72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000131fe0 sp=0xc000131fc8 pc=0x3bc5305\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000131fe8 sp=0xc000131fe0 pc=0x34fbcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 165\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x34fdae3\nrdx    0x0\nrdi    0xdba6cd0\nrsi    0x80\nrbp    0x7ffe490bad98\nrsp    0x7ffe490bad50\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x5\nr13    0x1\nr14    0xdba4be0\nr15    0x5\nrip    0x34fdae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"f5066ff63e6376032b0c7148b1a9dc1464444bc07cd3e831955ab47caf6b846d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047741844443369472","occurred_at":"2026-04-24T18:17:55Z","failed_at":"e2e","signature_id":"9a53c71edfc949be9d2dc81bb8243a9e3703f4e1ab90c33a165336c9cff54109","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016815f0\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kpr8f5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kpr8f5\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016815f0\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kpr8f5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kpr8f5\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-kpr8f5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-kpr8f5\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":29,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44","c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":29,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-24T04:34:07Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"5869f694919b57f396783de5fb5e5aac63b6ed404a34c7d0704f04932d6b76d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","occurred_at":"2026-04-24T04:34:07Z","failed_at":"e2e","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal"}},{"row_id":"59d29db764c4fdbfd9e1dd8e2dbd66017560c4009d1c98e90ebaec2e1eb354c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","occurred_at":"2026-04-24T04:34:07Z","failed_at":"e2e","signature_id":"8f5944fc31ef82fc3165f013626b18f0d15de769e768b4f09402c923662fe429","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2713.441s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2713.441s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"bc3d0aa2ee695cb41dd38b5d394dacd67e972b237d856c70d15167a7cff8ad36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","occurred_at":"2026-04-24T04:34:07Z","failed_at":"e2e","signature_id":"6c0108a49b790e72d13a010db752444cef14c3dc181ef82e5a4f8965cf12b337","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000ae7da0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-k6db5s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/195989b9-9d82-430b-b464-5e8928304bdb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/195989b9-9d82-430b-b464-5e8928304bdb\",\n      \"name\": \"195989b9-9d82-430b-b464-5e8928304bdb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-24T05:01:33.488341598Z\",\n      \"endTime\": \"2026-04-24T05:02:10.881244554Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000ae7da0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-k6db5s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/195989b9-9d82-430b-b464-5e8928304bdb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/195989b9-9d82-430b-b464-5e8928304bdb\",\n      \"name\": \"195989b9-9d82-430b-b464-5e8928304bdb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-24T05:01:33.488341598Z\",\n      \"endTime\": \"2026-04-24T05:02:10.881244554Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ffc159967e8f328ba908bc30fcc0643b30921ae247f91f48c528f6635a89b258","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047534526858530816","occurred_at":"2026-04-24T04:34:07Z","failed_at":"e2e","signature_id":"5ba612374b88287303367dd87819d840ddf45795ac363eda9583186c3d10f7a1","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e179e0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-c4hn4p, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-c4hn4p\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-c4hn4p, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e179e0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-c4hn4p, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-c4hn4p\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-c4hn4p, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-24T02:00:00Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"0164c165dc03f96a6bdac028073632be233e7e9f0f6ee960b097245eef870da3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","failed_at":"e2e","signature_id":"06f1523bb4cae24b536a5ddfd4e55a839d9f14750c796f75c0c08736e3ddb655","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000af28d0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-fvxs4h\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-fvxs4h, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000af28d0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-fvxs4h\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-fvxs4h, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"3829588e6017942f5e3c017ad8f2e9a3478eb165713332de447854c9a7f2a060","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","failed_at":"e2e","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal"}},{"row_id":"5de275b7d4394f5007ed8697a378f5f352c0827af86adcf3f94f28bb4c771e53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","failed_at":"e2e","signature_id":"87a3c2a4d30f66aa511a30e792b6f5f5ab20e2ad4e07b5b342e1e298e5e7acfc","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f80690\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-s5lfdk, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-s5lfdk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-s5lfdk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f80690\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-s5lfdk, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-s5lfdk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-s5lfdk, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"75372856ef7bb4c19e9e8839c3a01e759c91421f9a91681f012b3b357917efb8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","failed_at":"e2e","signature_id":"c158c05d163e3bf5e103dada36b1956e1c81456aa6ddb91e4f05bff6d96bf201","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015ac810\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-mfbprv, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-mfbprv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-mfbprv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015ac810\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-mfbprv, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-mfbprv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-mfbprv, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"79f5c936b887508aec17df6fe80b55d89d238667c23a607f27b6eb47ffc952bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","failed_at":"e2e","signature_id":"041e08e0f69fcee44ad56e8a175df8fa789ebf99065041189b005bda51546fdb","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Unexpected error:\n    \u003c*errors.joinError | 0xc000796660\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.p0l1s7v5v6c3i0l.4dkw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.p0l1s7v5v6c3i0l.4dkw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Unexpected error:\n    \u003c*errors.joinError | 0xc000796660\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.p0l1s7v5v6c3i0l.4dkw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.p0l1s7v5v6c3i0l.4dkw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.p0l1s7v5v6c3i0l.6i79.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io"}},{"row_id":"c7172a3868381268368791ed75286058cd03654735b64af8d679107a6b08c676","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","failed_at":"e2e","signature_id":"5890b5d4d3359e85a9cfea082995925e8cbe4f2930a0a58ae2ccd36d56f2f911","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009fa510\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7bq72q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-7bq72q, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009fa510\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7bq72q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-7bq72q, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mj8bzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-mj8bzq, error: context deadline exceeded"}},{"row_id":"cd405cffd0f1562763372a792df381e59298bc6690ea8c28d33e8ea8516d516c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047495744444698624","occurred_at":"2026-04-24T02:00:00Z","failed_at":"e2e","signature_id":"23cc9dae185dfede24507bc59941bd312c14aa6e0d3da7482a07abc571f45daf","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: Unexpected error:\n    \u003c*errors.joinError | 0xc001156c00\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: Unexpected error:\n    \u003c*errors.joinError | 0xc001156c00\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.n7a3d5c5g6v2q9o.plmx.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.n7a3d5c5g6v2q9o.vrfd.uksouth.aroapp-hcp.io"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}