{"meta":{"date":"2026-04-14","timezone":"UTC","generated_at":"2026-08-30T08:33:39Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":6,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044063738033606656","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-14T14:42:26Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dc94365c6958fc734395d9684677e6e1073b4366f3877a32456e2b562f4372ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044063738033606656","occurred_at":"2026-04-14T14:42:26Z","failed_at":"e2e","signature_id":"93dfef2dfe3a7d7a66cfae0829d14e3c4e0ca9720f4cafb98b26dacfc759d052","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc0006b7ab0\u003e: \n    expected 4 nodes, found 5\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc0006b7ab0\u003e: \n    expected 4 nodes, found 5\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe","failure_pattern":"expected 4 nodes, found 5","search_query":"expected 4 nodes, found 5"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043992098012663808","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-14T09:57:46Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"114c2eae881dc039dc340fa8653a458091d415168de03157aaa754abded5ea67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043992098012663808","occurred_at":"2026-04-14T09:57:46Z","failed_at":"e2e","signature_id":"f593007505cf5b70d80a7b459f445d87711dd06d2a9d5e5e1cef595f53023e90","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc0008f1050\u003e: \n    expected 4 nodes, found 5\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc0008f1050\u003e: \n    expected 4 nodes, found 5\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe","failure_pattern":"expected 4 nodes, found 5","search_query":"expected 4 nodes, found 5"}},{"row_id":"a3f28ad664ebccd5ed9e2b42c137f2cb346cde273d3fadabcde078ee84dae568","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043992098012663808","occurred_at":"2026-04-14T09:57:46Z","failed_at":"e2e","signature_id":"58cf1531eb9640b51b6e17d4735bb00bfa45f8aa5e2b3b1dacdb3b8347af3449","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001012960\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-965gz6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-965gz6\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001012960\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-965gz6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-965gz6\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-8qlspn, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8qlspn\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043959110528929792","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-14T07:46:41Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fcb9c9a4e0e1f7b0ab20844e023fd969b60956bf3a197e4d55e16b7d76e8f954","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043959110528929792","occurred_at":"2026-04-14T07:46:41Z","failed_at":"e2e","signature_id":"be73cf7c40b430fde3f0ef35ba12732c193a69781c2ebeb11acd87dda1105650","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e00090\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-8qlspn, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8qlspn\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e00090\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-8qlspn, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8qlspn\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-8qlspn, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8qlspn\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043954246331666432","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-14T07:27:21Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-14T03:36:54Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"0e9c9e468ffe8c8e606128497ddb767a260cd5f6fab48829029167fa24b0c81c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"2e7ae8a5e850d2a3c032169b72a31751be2e0f527794d321c98113d17864f08c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095ee70\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-98sfgl, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-98sfgl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-98sfgl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095ee70\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-98sfgl, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-98sfgl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-98sfgl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"1857b646bdcd1557715e102c1f6e17c471e40adf278d490f6e577505846eb8fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"6c9dd36bb3acf90a09e3b389719ccbfc456b382c4b0b208b71d37810afb91904","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001709110\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-plxw8q, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-plxw8q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-plxw8q, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001709110\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-plxw8q, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-plxw8q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-plxw8q, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"1be7b246d998c8c572253dbbf07366bb12ae078312286f79a518f92724c29554","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"5cc114e6e7a053dc86a72beebe5f5c5896f19e32bd493864ca526ca9467c3dd1","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00092d200\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5fppd9, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5fppd9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5fppd9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00092d200\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5fppd9, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5fppd9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5fppd9, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"1ee2b38217aae5302645cbb72bca2857daf0bcc650ce87532e334a663bc94bf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"c7dc0961ef7a7f892cf04ac35998a09091a19ec1209a2e321b05f8dbf81225ba","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000953290\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-rgk47s, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-rgk47s\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-rgk47s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000953290\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-rgk47s, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-rgk47s\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-rgk47s, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"26eb10dcbf52372957421ca2704bf07437fc242e723d7913b1e0712c05b097c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"563183ee5d10920ee8532879e0d788b8ff89b978b69f94015deb6283d3df7583","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00111ea80\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-n6pvrg, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-n6pvrg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-n6pvrg, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00111ea80\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-n6pvrg, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-n6pvrg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-n6pvrg, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"35625766fd52f7ddad41a0a2abdfe435a187da05d5054ad94f531178f1f9422a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"d652dd8263bd003e92425049a5e2c429f170d5b790f0d20b432d7e71687c7ebf","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001788a80\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-6n59wk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6n59wk in resource group rg-zstream-upgrade-4-19-4p2wmj, error: failed waiting for cluster=\"cluster-zstream-4-19-6n59wk\" in resourcegroup=\"rg-zstream-upgrade-4-19-4p2wmj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6n59wk in resource group rg-zstream-upgrade-4-19-4p2wmj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001788a80\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-6n59wk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6n59wk in resource group rg-zstream-upgrade-4-19-4p2wmj, error: failed waiting for cluster=\"cluster-zstream-4-19-6n59wk\" in resourcegroup=\"rg-zstream-upgrade-4-19-4p2wmj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6n59wk in resource group rg-zstream-upgrade-4-19-4p2wmj, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"6015b73b018fc9b5c69c4a1273f68f8a34d406513ab7611a822b7a7993ef5eab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"9898632942b623a60c408bf5037c45213cd29572bdd4d65f90843d7b7154106b","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a0ba0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-nv6nv7, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-nv6nv7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-nv6nv7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a0ba0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-nv6nv7, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-nv6nv7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-nv6nv7, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"6552ad6048cea36541dc3c75c2648360a308a670eaf8fbdf82a82d805a16a722","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"1a9be2463205e8af6f771592e46be3c73ba71f6c663b62eff22bc254486e4074","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014fcde0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014fcde0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"8388ab589fa076f0fbb9015f45e6b60fd5585e7c97602120e1fc8c89010c2c66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"29131c1b155c2db6cf23a92cea1e008533bcc6b683cc8b80ba427c5edbd0e134","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:117]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010b65d0\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-wr7qgw, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-wr7qgw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-wr7qgw, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:117]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010b65d0\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-wr7qgw, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-wr7qgw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-wr7qgw, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"a66a8fd77affd6aed77248dc9db525f854acb6b98b6326cf2ca4feacd45a8aa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"7673b223a59fdb2c6c551972eb0c1003f8a2e05b586d136586012475a9ab6e23","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a2ea0\u003e: \n    failed to create HCP cluster ea-listpcn8ws, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listpcn8ws in resource group ea-list-gtmnmh, error: failed waiting for cluster=\"ea-listpcn8ws\" in resourcegroup=\"ea-list-gtmnmh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listpcn8ws in resource group ea-list-gtmnmh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a2ea0\u003e: \n    failed to create HCP cluster ea-listpcn8ws, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listpcn8ws in resource group ea-list-gtmnmh, error: failed waiting for cluster=\"ea-listpcn8ws\" in resourcegroup=\"ea-list-gtmnmh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listpcn8ws in resource group ea-list-gtmnmh, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}},{"row_id":"d43d041e39d9862e5d7a1062b69eee6af9bccb7dcdf32bbb95ebb439aed0f3aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043896250377441280","occurred_at":"2026-04-14T03:36:54Z","failed_at":"e2e","signature_id":"42fe9a1e2b54f21c02ddcab554fc03ede8ca95599e0982fa5969be14a8da2f6b","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:99]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011f4ae0\u003e: \n    failed to create HCP cluster hcp-cluster-update-nodes, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-update-nodes in resource group rg-update-nodes-pfkhlf, error: failed waiting for cluster=\"hcp-cluster-update-nodes\" in resourcegroup=\"rg-update-nodes-pfkhlf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-update-nodes in resource group rg-update-nodes-pfkhlf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:99]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011f4ae0\u003e: \n    failed to create HCP cluster hcp-cluster-update-nodes, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-update-nodes in resource group rg-update-nodes-pfkhlf, error: failed waiting for cluster=\"hcp-cluster-update-nodes\" in resourcegroup=\"rg-update-nodes-pfkhlf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-update-nodes in resource group rg-update-nodes-pfkhlf, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-bc85j4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-bc85j4, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043872028859043840","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-14T02:00:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}