{"meta":{"date":"2026-07-08","timezone":"UTC","generated_at":"2026-08-28T08:29:08Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":4,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2074992017036808192","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-08T23:00:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-08T14:15:53Z"},"failed_at":["e2e"],"failed_test_count":44,"occurrences":[{"row_id":"08cac5ad2dacca25d4dc41754b283410abc4a6024498136c6cd97f9cd58b01f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"9900df039603a54ab01e168fa49161f5f25e123cb747c63234da923a4825d437","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create HCP cluster np-skip-minor-d4skmk at version 4.22.3\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00184a4b0\u003e: \n    failed to create HCP cluster np-skip-minor-d4skmk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-skip-minor-d4skmk in resource group rg-np-skip-minor-d4skmk-bms7sf, error: failed waiting for cluster=\"np-skip-minor-d4skmk\" in resourcegroup=\"rg-np-skip-minor-d4skmk-bms7sf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-skip-minor-d4skmk in resource group rg-np-skip-minor-d4skmk-bms7sf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create HCP cluster np-skip-minor-d4skmk at version 4.22.3\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00184a4b0\u003e: \n    failed to create HCP cluster np-skip-minor-d4skmk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-skip-minor-d4skmk in resource group rg-np-skip-minor-d4skmk-bms7sf, error: failed waiting for cluster=\"np-skip-minor-d4skmk\" in resourcegroup=\"rg-np-skip-minor-d4skmk-bms7sf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-skip-minor-d4skmk in resource group rg-np-skip-minor-d4skmk-bms7sf, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"08cb6346ded0cc1af0bff577b3a3d3b26a62140ef1cc856b89ca2229fb53c9d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"051cd11894b5bb1761f42390833b9dfa4b35559010eb57af3a4752028484c08e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011863f0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-b5lpzp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-b5lpzp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011863f0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-b5lpzp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-b5lpzp, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"08e35db55ddb68c1dafe4a3e0fdaba0ade836cbbfc5ec9d938b740f2248fd6f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"8d2fefd28378eea263a122e791f5bf24be54b891eac45c0b9828883a6d2bd4ca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-m25z2g/cluster-candidate-4-21-vz9lnx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a18360\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vz9lnx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-vz9lnx in resource group rg-candidate-4-21-m25z2g, error: failed waiting for cluster=\"cluster-candidate-4-21-vz9lnx\" in resourcegroup=\"rg-candidate-4-21-m25z2g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-vz9lnx in resource group rg-candidate-4-21-m25z2g, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-m25z2g/cluster-candidate-4-21-vz9lnx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a18360\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vz9lnx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-vz9lnx in resource group rg-candidate-4-21-m25z2g, error: failed waiting for cluster=\"cluster-candidate-4-21-vz9lnx\" in resourcegroup=\"rg-candidate-4-21-m25z2g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-vz9lnx in resource group rg-candidate-4-21-m25z2g, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"163fdfa8e68a991083cc8b6c74c7e954d961d1723fd97227957c866f906f6cf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"c44be3b2c23e0b21e01640dee289704e25248b5de84c0d9af6a9ef4e3622863c","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000be54a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-fw6j67, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-fw6j67\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-fw6j67, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000be54a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-fw6j67, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-fw6j67\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-fw6j67, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"1a0f357e5c2fab26d96549bde81e77ed780047b554576c940d3d6f212ed1f0ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"b43dbfa46db277cbbecd4363e137ad8177c0506fabbad6467bc1628c34e1f2af","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-gsgnc4\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001540870\u003e: \n    failed to create HCP cluster np-noedge-gsgnc4, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-noedge-gsgnc4 in resource group rg-np-noedge-gsgnc4-8q29l4, error: failed waiting for cluster=\"np-noedge-gsgnc4\" in resourcegroup=\"rg-np-noedge-gsgnc4-8q29l4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-noedge-gsgnc4 in resource group rg-np-noedge-gsgnc4-8q29l4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-gsgnc4\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001540870\u003e: \n    failed to create HCP cluster np-noedge-gsgnc4, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-noedge-gsgnc4 in resource group rg-np-noedge-gsgnc4-8q29l4, error: failed waiting for cluster=\"np-noedge-gsgnc4\" in resourcegroup=\"rg-np-noedge-gsgnc4-8q29l4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-noedge-gsgnc4 in resource group rg-np-noedge-gsgnc4-8q29l4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"1b8b6b4c945980d77da8d56369fa754924d7480184f3c42cea0ec4fb492cbb1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"0c8f8984d19fca5e2109d89426e8465f02c21a95d245e2bfffecaacafe077a2a","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010e6930\u003e: \n    failed to create HCP cluster ea-listbk9qsn, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listbk9qsn in resource group ea-list-8krq8c, error: failed waiting for cluster=\"ea-listbk9qsn\" in resourcegroup=\"ea-list-8krq8c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listbk9qsn in resource group ea-list-8krq8c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010e6930\u003e: \n    failed to create HCP cluster ea-listbk9qsn, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listbk9qsn in resource group ea-list-8krq8c, error: failed waiting for cluster=\"ea-listbk9qsn\" in resourcegroup=\"ea-list-8krq8c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listbk9qsn in resource group ea-list-8krq8c, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"1e416e4b74fbebc231309568d475522ffae875025d37595a87342e8ed1f40447","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"d85ae202e1ac677177b74e124a438429fcb961efc8ca69cecbd30c4eaf4703c7","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a6840\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-kx68zb, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kx68zb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-kx68zb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a6840\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-kx68zb, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-kx68zb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-kx68zb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"24373c6ce4200899bdaf6f06614f1f018721a2500554599709d624d2d92e80b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"a53592d211b24d3b93fc9adc6cda79a9440cbfcccf2a355f61ad6d89d25aeb7e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001546b40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-9jqgqm, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9jqgqm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-9jqgqm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001546b40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-9jqgqm, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-9jqgqm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-9jqgqm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"30ec188c2f1546e4b386aef5598807a2f3e0173874881720d23520fec1b9a3e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"b1890c93497df6ed0faf59443ecdf52d709a1cd29a82ebdff4355fe10f069cc9","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-hc2cbq with version 4.21.22\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00178c090\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-hc2cbq, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-hc2cbq in resource group rg-np-version-upgrade-hc2cbq-9ltbgw, error: failed waiting for cluster=\"np-version-upgrade-cluster-hc2cbq\" in resourcegroup=\"rg-np-version-upgrade-hc2cbq-9ltbgw\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-hc2cbq in resource group rg-np-version-upgrade-hc2cbq-9ltbgw, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-hc2cbq with version 4.21.22\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00178c090\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-hc2cbq, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-hc2cbq in resource group rg-np-version-upgrade-hc2cbq-9ltbgw, error: failed waiting for cluster=\"np-version-upgrade-cluster-hc2cbq\" in resourcegroup=\"rg-np-version-upgrade-hc2cbq-9ltbgw\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-hc2cbq in resource group rg-np-version-upgrade-hc2cbq-9ltbgw, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"36f90eb29d66cad085f4ff5c33fac5c3129e205fb76e5f87e3816183c3742eb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"472762e9ab805bceb9d643d5e2d3af71ae642684b01bd95e5061e899dc465a00","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a794d0\u003e: \n    failed to create HCP cluster np-downgrade-mtztsg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-mtztsg in resource group rg-np-downgrade-mtztsg-7wplqk, error: failed waiting for cluster=\"np-downgrade-mtztsg\" in resourcegroup=\"rg-np-downgrade-mtztsg-7wplqk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-mtztsg in resource group rg-np-downgrade-mtztsg-7wplqk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a794d0\u003e: \n    failed to create HCP cluster np-downgrade-mtztsg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-mtztsg in resource group rg-np-downgrade-mtztsg-7wplqk, error: failed waiting for cluster=\"np-downgrade-mtztsg\" in resourcegroup=\"rg-np-downgrade-mtztsg-7wplqk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-mtztsg in resource group rg-np-downgrade-mtztsg-7wplqk, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"3738eb9894f503f05e04b243329154608b35c94f1c723611a2616447f920cb3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"aed07da93803005b46e26e75e613559489562755fe1f89333bd605c689f5405d","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000ecdfe0 sp=0xc000ecdfc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000ecdfe8 sp=0xc000ecdfe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 282\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 315 gp=0xc0016b9340 m=nil [select]:\nruntime.gopark(0xc000b05f48?, 0x2?, 0xc0?, 0x5d?, 0xc000b05ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000b05d78 sp=0xc000b05d58 pc=0x2f3650e\nruntime.selectgo(0xc000b05f48, 0xc000b05ef0, 0xc000e26c40?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000b05eb8 sp=0xc000b05d78 pc=0x2f112f7\nnet/http.(*persistConn).writeLoop(0xc000adefc0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000b05fc8 sp=0xc000b05eb8 pc=0x32fec87\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000b05fe0 sp=0xc000b05fc8 pc=0x32fb925\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b05fe8 sp=0xc000b05fe0 pc=0x2f3ecc1\ncreated by net/http.(*Transport).dialConn in goroutine 283\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 314 gp=0xc0016b96c0 m=nil [IO wait]:\nruntime.gopark(0xc0009cc3a0?, 0xc0000c3788?, 0xc5?, 0x3e?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0000c3768 sp=0xc0000c3748 pc=0x2f3650e\nruntime.netpollblock(0x2fa0ad8?, 0x2ec2066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000c37a0 sp=0xc0000c3768 pc=0x2ef5fb7\ninternal/poll.runtime_pollWait(0x7f0ad6814a00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0000c37c0 sp=0xc0000c37a0 pc=0x2f34da5\ninternal/poll.(*pollDesc).wait(0xc001816c80?, 0xc000dec000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000c37e8 sp=0xc0000c37c0 pc=0x2fc3907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc001816c80, {0xc000dec000, 0x1300, 0x1300})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0000c3880 sp=0xc0000c37e8 pc=0x2fc4d39\nnet.(*netFD).Read(0xc001816c80, {0xc000dec000?, 0xc000dec000?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0000c38c8 sp=0xc0000c3880 pc=0x31893c5\nnet.(*conn).Read(0xc0000b4318, {0xc000dec000?, 0x7f0ad57c8a48?, 0x7f0b1d3fb5c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc0000c3910 sp=0xc0000c38c8 pc=0x319b465\ncrypto/tls.(*atLeastReader).Read(0xc0010f29d8, {0xc000dec000?, 0x120a?, 0x2ed79d6?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc0000c3958 sp=0xc0000c3910 pc=0x3206ffb\nbytes.(*Buffer).ReadFrom(0xc0009a82a8, {0x92dd8a0, 0xc0010f29d8})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0000c39b0 sp=0xc0000c3958 pc=0x3007578\ncrypto/tls.(*Conn).readFromUntil(0xc0009a8008, {0x92df380, 0xc0000b4318}, 0xc0000c3a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0000c39e8 sp=0xc0000c39b0 pc=0x32071de\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0009a8008, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc0000c3c60 sp=0xc0000c39e8 pc=0x320425b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0009a8008, {0xc000b0e000, 0x1000, 0xc000a4f650?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0000c3cc8 sp=0xc0000c3c60 pc=0x320ab65\nnet/http.(*persistConn).Read(0xc000adefc0, {0xc000b0e000?, 0x32f4be5?, 0x92d8260?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc0000c3d28 sp=0xc0000c3cc8 pc=0x32fc4e7\nbufio.(*Reader).fill(0xc000c2daa0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc0000c3d60 sp=0xc0000c3d28 pc=0x309ca23\nbufio.(*Reader).Peek(0xc000c2daa0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc0000c3d80 sp=0xc0000c3d60 pc=0x309cbb3\nnet/http.(*persistConn).readLoop(0xc000adefc0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0000c3fc8 sp=0xc0000c3d80 pc=0x32fcff2\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0000c3fe0 sp=0xc0000c3fc8 pc=0x32fb985\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c3fe8 sp=0xc0000c3fe0 pc=0x2f3ecc1\ncreated by net/http.(*Transport).dialConn in goroutine 283\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax    0xca\nrbx    0x0\nrcx    0x2f40ae3\nrdx    0x0\nrdi    0xc9bba10\nrsi    0x80\nrbp    0x7fff7dbe62d0\nrsp    0x7fff7dbe6288\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x0\nr13    0x0\nr14    0xc9b9480\nr15    0x7\nrip    0x2f40ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000ecdfe0 sp=0xc000ecdfc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000ecdfe8 sp=0xc000ecdfe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 282\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 315 gp=0xc0016b9340 m=nil [select]:\nruntime.gopark(0xc000b05f48?, 0x2?, 0xc0?, 0x5d?, 0xc000b05ef4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000b05d78 sp=0xc000b05d58 pc=0x2f3650e\nruntime.selectgo(0xc000b05f48, 0xc000b05ef0, 0xc000e26c40?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000b05eb8 sp=0xc000b05d78 pc=0x2f112f7\nnet/http.(*persistConn).writeLoop(0xc000adefc0)\n\t/usr/lib/golang/src/net/http/transport.go:2600 +0xe7 fp=0xc000b05fc8 sp=0xc000b05eb8 pc=0x32fec87\nnet/http.(*Transport).dialConn.gowrap3()\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x25 fp=0xc000b05fe0 sp=0xc000b05fc8 pc=0x32fb925\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000b05fe8 sp=0xc000b05fe0 pc=0x2f3ecc1\ncreated by net/http.(*Transport).dialConn in goroutine 283\n\t/usr/lib/golang/src/net/http/transport.go:1948 +0x17a5\n\ngoroutine 314 gp=0xc0016b96c0 m=nil [IO wait]:\nruntime.gopark(0xc0009cc3a0?, 0xc0000c3788?, 0xc5?, 0x3e?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0000c3768 sp=0xc0000c3748 pc=0x2f3650e\nruntime.netpollblock(0x2fa0ad8?, 0x2ec2066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000c37a0 sp=0xc0000c3768 pc=0x2ef5fb7\ninternal/poll.runtime_pollWait(0x7f0ad6814a00, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc0000c37c0 sp=0xc0000c37a0 pc=0x2f34da5\ninternal/poll.(*pollDesc).wait(0xc001816c80?, 0xc000dec000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000c37e8 sp=0xc0000c37c0 pc=0x2fc3907\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc001816c80, {0xc000dec000, 0x1300, 0x1300})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0000c3880 sp=0xc0000c37e8 pc=0x2fc4d39\nnet.(*netFD).Read(0xc001816c80, {0xc000dec000?, 0xc000dec000?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc0000c38c8 sp=0xc0000c3880 pc=0x31893c5\nnet.(*conn).Read(0xc0000b4318, {0xc000dec000?, 0x7f0ad57c8a48?, 0x7f0b1d3fb5c0?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc0000c3910 sp=0xc0000c38c8 pc=0x319b465\ncrypto/tls.(*atLeastReader).Read(0xc0010f29d8, {0xc000dec000?, 0x120a?, 0x2ed79d6?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc0000c3958 sp=0xc0000c3910 pc=0x3206ffb\nbytes.(*Buffer).ReadFrom(0xc0009a82a8, {0x92dd8a0, 0xc0010f29d8})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc0000c39b0 sp=0xc0000c3958 pc=0x3007578\ncrypto/tls.(*Conn).readFromUntil(0xc0009a8008, {0x92df380, 0xc0000b4318}, 0xc0000c3a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0000c39e8 sp=0xc0000c39b0 pc=0x32071de\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0009a8008, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc0000c3c60 sp=0xc0000c39e8 pc=0x320425b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0009a8008, {0xc000b0e000, 0x1000, 0xc000a4f650?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0000c3cc8 sp=0xc0000c3c60 pc=0x320ab65\nnet/http.(*persistConn).Read(0xc000adefc0, {0xc000b0e000?, 0x32f4be5?, 0x92d8260?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc0000c3d28 sp=0xc0000c3cc8 pc=0x32fc4e7\nbufio.(*Reader).fill(0xc000c2daa0)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc0000c3d60 sp=0xc0000c3d28 pc=0x309ca23\nbufio.(*Reader).Peek(0xc000c2daa0, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc0000c3d80 sp=0xc0000c3d60 pc=0x309cbb3\nnet/http.(*persistConn).readLoop(0xc000adefc0)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc0000c3fc8 sp=0xc0000c3d80 pc=0x32fcff2\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc0000c3fe0 sp=0xc0000c3fc8 pc=0x32fb985\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c3fe8 sp=0xc0000c3fe0 pc=0x2f3ecc1\ncreated by net/http.(*Transport).dialConn in goroutine 283\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\nrax    0xca\nrbx    0x0\nrcx    0x2f40ae3\nrdx    0x0\nrdi    0xc9bba10\nrsi    0x80\nrbp    0x7fff7dbe62d0\nrsp    0x7fff7dbe6288\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x0\nr13    0x0\nr14    0xc9b9480\nr15    0x7\nrip    0x2f40ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"3a354a0957dda3b4df9bba8fb8d40bff1bb5cf8c050fc5e9e5852f0d210d0f2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"7b311f6b09c3bb8e223e61a1567cbba9f44bc8fab49954456fd9328e616945db","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state"}},{"row_id":"43f5881cd59106b6890d1f7e21984cbd90f692643e786f51dfc763a45f8feb9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"28e8b642415a41acf0353096baf5f7d28fecfdc10c57942eb496505bf86fa40a","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:111]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013d0a50\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7bs4rv, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7bs4rv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7bs4rv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:111]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013d0a50\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7bs4rv, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7bs4rv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7bs4rv, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"4448407f1c53981828d53310489df5fc9697444a32f8c723908038196b5da81c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"4627af1e23c7ebdc066d9a600aebc5ab11f5a83fe29e8960f9435305e05b7490","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfd0e0\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cx-rg-hcp-cluster in resource group cx-rg-cluster-wf5j2r, error: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-wf5j2r\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cx-rg-hcp-cluster in resource group cx-rg-cluster-wf5j2r, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfd0e0\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cx-rg-hcp-cluster in resource group cx-rg-cluster-wf5j2r, error: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-wf5j2r\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cx-rg-hcp-cluster in resource group cx-rg-cluster-wf5j2r, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"4a08016e6527f179ca3d113e2d83a138efae53ed44e1b728e79dc00783c3d95a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"a015ba8d7eed4e159361d381519baece2f756c8b01ac5da13c659a761bf7cea9","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001730060\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fpqwh2, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-fpqwh2\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fpqwh2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001730060\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fpqwh2, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-fpqwh2\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fpqwh2, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"4b7980c9d6b2a14a6fa2fb1ce3449da98d1866a987db698fe698f19e747aea42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"6849a249e6c2ebcce32d57465d8d5de70673fee2796ccc6549b8f68949d8a30f","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-xzshn6 with version 4.21.22\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a81f20\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-xzshn6, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-xzshn6 in resource group rg-np-version-upgrade-xzshn6-nkwngk, error: failed waiting for cluster=\"np-version-upgrade-cluster-xzshn6\" in resourcegroup=\"rg-np-version-upgrade-xzshn6-nkwngk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-xzshn6 in resource group rg-np-version-upgrade-xzshn6-nkwngk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-xzshn6 with version 4.21.22\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a81f20\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-xzshn6, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-xzshn6 in resource group rg-np-version-upgrade-xzshn6-nkwngk, error: failed waiting for cluster=\"np-version-upgrade-cluster-xzshn6\" in resourcegroup=\"rg-np-version-upgrade-xzshn6-nkwngk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-xzshn6 in resource group rg-np-version-upgrade-xzshn6-nkwngk, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"65182feaf42d173d53bdf04f4cbf2f195e60208bb4ffdcda129b80a7cda19f8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"a9d3a543b7b2dfdd016d74d3237526fcf910d2eb7f51b885db44ff04acfd21aa","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0024cf320\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-c52gps, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c52gps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-c52gps, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0024cf320\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-c52gps, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c52gps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-c52gps, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"6641ce7a3532c5cd28fa4cb4f216c6eb02026170e46e15b663413209d79c3626","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"23a2fa0bad2ec2e2cb932442d27b04f8e284933fd3cd9c6ac247daf6683b23c9","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007000f0\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-tqqwkj, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-tqqwkj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-tqqwkj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007000f0\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-tqqwkj, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-tqqwkj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-tqqwkj, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"689ff6e5259fe5227ee90af9a2eaf66b233a12a6e0eeea6b7f6d991119bc4ea8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"31e93372abd82a72fc3f31e83198cc64b075fc70d727f16d56929fee1343e2fa","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-sgttzb\" with version 4.22.1 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a20cc0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-sgttzb, caused by: timeout '35.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-sgttzb in resource group rg-zstream-upgrade-4-22-wxptvz, error: failed waiting for cluster=\"cluster-zstream-4-22-sgttzb\" in resourcegroup=\"rg-zstream-upgrade-4-22-wxptvz\" to finish creating, caused by: timeout '35.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-sgttzb in resource group rg-zstream-upgrade-4-22-wxptvz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-sgttzb\" with version 4.22.1 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a20cc0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-sgttzb, caused by: timeout '35.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-sgttzb in resource group rg-zstream-upgrade-4-22-wxptvz, error: failed waiting for cluster=\"cluster-zstream-4-22-sgttzb\" in resourcegroup=\"rg-zstream-upgrade-4-22-wxptvz\" to finish creating, caused by: timeout '35.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-sgttzb in resource group rg-zstream-upgrade-4-22-wxptvz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"6f353cfbbe5703913bd2f1ec353d2345b8ce3c6009cbbc9ad7c468df914f7d15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"fa82b432fd7a36953b4c71f37791bced4d6db1832370f07a5c77508bfa36ef7e","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79232f7fe5f1a65f3af30b7ba05ac2a1643a1a699ce682f3bc11178c03f0cbf4","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation (timeout '20.000000' minutes)","search_query":"context deadline exceeded"}},{"row_id":"7a33ae83d47114160a06885a46015bea287f695fac1495c42259d3336dbee3cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"33312dbc5fc68ebce09e9753f96df864fb727489a2e4bab2e61366b0d13c5f91","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"c000ea0748)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc00158dac8 sp=0xc00158daa0 pc=0x2f5b405\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000ea0740)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc00158dae8 sp=0xc00158dac8 pc=0x4a8aabe\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610({0x7f130446f260, 0xc00157e930}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:525 +0x8de fp=0xc00158dd90 sp=0xc00158dae8 pc=0x6c8efde\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000e4c840, {0x7f130446f260, 0xc00157e930}, {0xc000dc6ea0, 0x22}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x29d fp=0xc00158df08 sp=0xc00158dd90 pc=0x6cb215d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f130446f260?, 0xc00157e930?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc00158dfc0 sp=0xc00158df08 pc=0x6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc00158dfe0 sp=0xc00158dfc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00158dfe8 sp=0xc00158dfe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 517\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 533 gp=0xc001103340 m=nil [select]:\nruntime.gopark(0xc00189d768?, 0x2?, 0x0?, 0x40?, 0xc00189d75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00189d5e8 sp=0xc00189d5c8 pc=0x2f3650e\nruntime.selectgo(0xc00189d768, 0xc00189d758, 0xc000a80180?, 0x0, 0x370521c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00189d728 sp=0xc00189d5e8 pc=0x2f112f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x9376198, 0xc000033490}, 0x8779154?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00189d798 sp=0xc00189d728 pc=0x3704976\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x93c8b00, {0x9376198, 0xc000033490}, 0x2f33af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00189da98 sp=0xc00189d798 pc=0x3e10172\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x9376198, 0xc000033490}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, {0xc0008e1ca0, 0xd})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc00189dc20 sp=0xc00189da98 pc=0x6c8bf85\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x9376198?, 0xc000033490?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc00189dcc8 sp=0xc00189dc20 pc=0x6c8b9b2\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x9376198?, 0xc000033490?}, 0xc0000334a0?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc00189dd20 sp=0xc00189dcc8 pc=0x36b7f42\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x9376198, 0xc000033490}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc00189de68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc00189dd98 sp=0xc00189dd20 pc=0x36b5fee\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x9376128, 0xc000e91b80}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, {0xc0008e1ca0, 0xd}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc00189def0 sp=0xc00189dd98 pc=0x6c8b725\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc00189df78 sp=0xc00189def0 pc=0x6c8f61e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00189dfe0 sp=0xc00189df78 pc=0x4a8ac10\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00189dfe8 sp=0xc00189dfe0 pc=0x2f3ecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 526\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x2f40ae3\nrdx    0x0\nrdi    0xc9bba10\nrsi    0x80\nrbp    0x7ffc382442f0\nrsp    0x7ffc382442a8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x3\nr13    0x1\nr14    0xc9b9480\nr15    0x7\nrip    0x2f40ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nc000ea0748)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc00158dac8 sp=0xc00158daa0 pc=0x2f5b405\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000ea0740)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc00158dae8 sp=0xc00158dac8 pc=0x4a8aabe\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610({0x7f130446f260, 0xc00157e930}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:525 +0x8de fp=0xc00158dd90 sp=0xc00158dae8 pc=0x6c8efde\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000e4c840, {0x7f130446f260, 0xc00157e930}, {0xc000dc6ea0, 0x22}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x29d fp=0xc00158df08 sp=0xc00158dd90 pc=0x6cb215d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f130446f260?, 0xc00157e930?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc00158dfc0 sp=0xc00158df08 pc=0x6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc00158dfe0 sp=0xc00158dfc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00158dfe8 sp=0xc00158dfe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 517\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 533 gp=0xc001103340 m=nil [select]:\nruntime.gopark(0xc00189d768?, 0x2?, 0x0?, 0x40?, 0xc00189d75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00189d5e8 sp=0xc00189d5c8 pc=0x2f3650e\nruntime.selectgo(0xc00189d768, 0xc00189d758, 0xc000a80180?, 0x0, 0x370521c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00189d728 sp=0xc00189d5e8 pc=0x2f112f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x9376198, 0xc000033490}, 0x8779154?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00189d798 sp=0xc00189d728 pc=0x3704976\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x93c8b00, {0x9376198, 0xc000033490}, 0x2f33af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00189da98 sp=0xc00189d798 pc=0x3e10172\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x9376198, 0xc000033490}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, {0xc0008e1ca0, 0xd})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc00189dc20 sp=0xc00189da98 pc=0x6c8bf85\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x9376198?, 0xc000033490?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc00189dcc8 sp=0xc00189dc20 pc=0x6c8b9b2\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x9376198?, 0xc000033490?}, 0xc0000334a0?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc00189dd20 sp=0xc00189dcc8 pc=0x36b7f42\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x9376198, 0xc000033490}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc00189de68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc00189dd98 sp=0xc00189dd20 pc=0x36b5fee\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x9376128, 0xc000e91b80}, 0xc000a2af30, {0xc000dc6ea0, 0x22}, {0xc0008e1ca0, 0xd}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc00189def0 sp=0xc00189dd98 pc=0x6c8b725\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc00189df78 sp=0xc00189def0 pc=0x6c8f61e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00189dfe0 sp=0xc00189df78 pc=0x4a8ac10\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00189dfe8 sp=0xc00189dfe0 pc=0x2f3ecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 526\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xca\nrbx    0x0\nrcx    0x2f40ae3\nrdx    0x0\nrdi    0xc9bba10\nrsi    0x80\nrbp    0x7ffc382442f0\nrsp    0x7ffc382442a8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x3\nr13    0x1\nr14    0xc9b9480\nr15    0x7\nrip    0x2f40ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"94f39f4237084b218e29fa4d9c67a9fd5151b82b671b997c8984addeaccf47b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"1ece47042c09ec8f35bdf130d7465d5968a130e2c070435d1837cee337207908","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016868d0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-66x82b, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-66x82b\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-66x82b, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016868d0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-66x82b, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-66x82b\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-66x82b, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"a0ad4c4d63d2dfda0a0b67720418bdef378c68beb539a53e0ce45c73ce93b28a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"d53160db757538538a7bb1d0f9f58b70bdbe22b7f497589b5857d723c14d6029","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a6a390\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-gmp2c9, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-gmp2c9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-gmp2c9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a6a390\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-gmp2c9, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-gmp2c9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-gmp2c9, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"af209e55b2e05fde36d3500dacd32dcd1b3e7bf6f5ce92ac9e082ce308c2eb33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"9f354ce5c9797aa06703b2e23cd0fc2fde85c09e8515cedfd2e2657ea7c923aa","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00128a780\u003e: \n    failed to create HCP cluster cilium-cl, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-2gfj2m, error: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-2gfj2m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-2gfj2m, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00128a780\u003e: \n    failed to create HCP cluster cilium-cl, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-2gfj2m, error: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-2gfj2m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-2gfj2m, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"b23de3793f85e12cfd73003888c27077e6b654b4082fa9c7fa74e49d2d8e1934","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"898093075b4a8bc027b4d5ed45992df6c20fed3b2737cdbc9c5ad690668b9a47","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008763c0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9pggn8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster fips-enabled-cluster in resource group fips-enabled-9pggn8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008763c0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-9pggn8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster fips-enabled-cluster in resource group fips-enabled-9pggn8, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"bd516b6ed27d3385fd77d7127225f787d28f71048ec4ea682f586ffded941d2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"29b36ea550191b186760d9cc3d69010148ad56d86051a11c874f2cdd88fef386","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-gb7zft\" with version 4.20.26 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001368ae0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-gb7zft, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-gb7zft in resource group rg-zstream-upgrade-4-20-ld7ldx, error: failed waiting for cluster=\"cluster-zstream-4-20-gb7zft\" in resourcegroup=\"rg-zstream-upgrade-4-20-ld7ldx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-gb7zft in resource group rg-zstream-upgrade-4-20-ld7ldx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-gb7zft\" with version 4.20.26 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001368ae0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-gb7zft, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-gb7zft in resource group rg-zstream-upgrade-4-20-ld7ldx, error: failed waiting for cluster=\"cluster-zstream-4-20-gb7zft\" in resourcegroup=\"rg-zstream-upgrade-4-20-ld7ldx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-gb7zft in resource group rg-zstream-upgrade-4-20-ld7ldx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"c0594ebfe2510f6a9eda4d756e024c6339b91c816be8b0db724219a1cb13c95a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"ed90acc1a208787b943d8fa885bcb7d65e2c641e8155aef811723d627a4dfb37","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"ync/waitgroup.go:206 +0x85 fp=0xc000e5fac8 sp=0xc000e5faa0 pc=0x2f5b405\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000e97840)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc000e5fae8 sp=0xc000e5fac8 pc=0x4a8aabe\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610({0x7fc97c1bf660, 0xc000b97f50}, 0xc001082750, {0xc0009bb5c0, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:525 +0x8de fp=0xc000e5fd90 sp=0xc000e5fae8 pc=0x6c8efde\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000c154a0, {0x7fc97c1bf660, 0xc000b97f50}, {0xc0009bb5c0, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x29d fp=0xc000e5ff08 sp=0xc000e5fd90 pc=0x6cb215d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7fc97c1bf660?, 0xc000b97f50?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000e5ffc0 sp=0xc000e5ff08 pc=0x6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000e5ffe0 sp=0xc000e5ffc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e5ffe8 sp=0xc000e5ffe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 470\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 491 gp=0xc0011ae380 m=nil [select]:\nruntime.gopark(0xc00165b768?, 0x2?, 0x0?, 0x5a?, 0xc00165b75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00165b5e8 sp=0xc00165b5c8 pc=0x2f3650e\nruntime.selectgo(0xc00165b768, 0xc00165b758, 0xc0006feca0?, 0x0, 0x370521c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00165b728 sp=0xc00165b5e8 pc=0x2f112f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x9376198, 0xc001566070}, 0x8779154?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00165b798 sp=0xc00165b728 pc=0x3704976\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x93c8b00, {0x9376198, 0xc001566070}, 0x2f33af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00165ba98 sp=0xc00165b798 pc=0x3e10172\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x9376198, 0xc001566070}, 0xc001082750, {0xc0009bb5c0, 0x11}, {0xc00154a300, 0x12})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc00165bc20 sp=0xc00165ba98 pc=0x6c8bf85\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x9376198?, 0xc001566070?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc00165bcc8 sp=0xc00165bc20 pc=0x6c8b9b2\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x9376198?, 0xc001566070?}, 0xc001566080?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc00165bd20 sp=0xc00165bcc8 pc=0x36b7f42\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x9376198, 0xc001566070}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc00165be68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc00165bd98 sp=0xc00165bd20 pc=0x36b5fee\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x9376128, 0xc0007baff0}, 0xc001082750, {0xc0009bb5c0, 0x11}, {0xc00154a300, 0x12}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc00165bef0 sp=0xc00165bd98 pc=0x6c8b725\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc00165bf78 sp=0xc00165bef0 pc=0x6c8f61e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00165bfe0 sp=0xc00165bf78 pc=0x4a8ac10\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00165bfe8 sp=0xc00165bfe0 pc=0x2f3ecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 502\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2eb5e2e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffcb9d27d24\nrbp    0x7ffcb9d27cd8\nrsp    0x7ffcb9d27c90\nr8     0x0\nr9     0x0\nr10    0x393\nr11    0x246\nr12    0x7ffcb9d27d68\nr13    0x1\nr14    0xc9b9480\nr15    0x3\nrip    0x2eb5e2e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nync/waitgroup.go:206 +0x85 fp=0xc000e5fac8 sp=0xc000e5faa0 pc=0x2f5b405\ngolang.org/x/sync/errgroup.(*Group).Wait(0xc000e97840)\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:56 +0x1e fp=0xc000e5fae8 sp=0xc000e5fac8 pc=0x4a8aabe\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610({0x7fc97c1bf660, 0xc000b97f50}, 0xc001082750, {0xc0009bb5c0, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:525 +0x8de fp=0xc000e5fd90 sp=0xc000e5fae8 pc=0x6c8efde\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000c154a0, {0x7fc97c1bf660, 0xc000b97f50}, {0xc0009bb5c0, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:537 +0x29d fp=0xc000e5ff08 sp=0xc000e5fd90 pc=0x6cb215d\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7fc97c1bf660?, 0xc000b97f50?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000e5ffc0 sp=0xc000e5ff08 pc=0x6cb01f1\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000e5ffe0 sp=0xc000e5ffc0 pc=0x6cb0068\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e5ffe8 sp=0xc000e5ffe0 pc=0x2f3ecc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 470\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\ngoroutine 491 gp=0xc0011ae380 m=nil [select]:\nruntime.gopark(0xc00165b768?, 0x2?, 0x0?, 0x5a?, 0xc00165b75c?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc00165b5e8 sp=0xc00165b5c8 pc=0x2f3650e\nruntime.selectgo(0xc00165b768, 0xc00165b758, 0xc0006feca0?, 0x0, 0x370521c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc00165b728 sp=0xc00165b5e8 pc=0x2f112f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x9376198, 0xc001566070}, 0x8779154?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc00165b798 sp=0xc00165b728 pc=0x3704976\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x93c8b00, {0x9376198, 0xc001566070}, 0x2f33af3)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc00165ba98 sp=0xc00165b798 pc=0x3e10172\ngithub.com/Azure/ARO-HCP/test/util/framework.deleteHCPClusterAttempt({0x9376198, 0xc001566070}, 0xc001082750, {0xc0009bb5c0, 0x11}, {0xc00154a300, 0x12})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:240 +0x365 fp=0xc00165bc20 sp=0xc00165ba98 pc=0x6c8bf85\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610.func1({0x9376198?, 0xc001566070?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:198 +0x72 fp=0xc00165bcc8 sp=0xc00165bc20 pc=0x6c8b9b2\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x9376198?, 0xc001566070?}, 0xc001566080?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc00165bd20 sp=0xc00165bcc8 pc=0x36b7f42\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x9376198, 0xc001566070}, {0xdf8475800, 0x4000000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc00165be68)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc00165bd98 sp=0xc00165bd20 pc=0x36b5fee\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteHCPCluster20240610({0x9376128, 0xc0007baff0}, 0xc001082750, {0xc0009bb5c0, 0x11}, {0xc00154a300, 0x12}, 0x274a48a7800)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:196 +0x285 fp=0xc00165bef0 sp=0xc00165bd98 pc=0x6c8b725\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteAllHCPClusters20240610.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:522 +0x7e fp=0xc00165bf78 sp=0xc00165bef0 pc=0x6c8f61e\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:93 +0x50 fp=0xc00165bfe0 sp=0xc00165bf78 pc=0x4a8ac10\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00165bfe8 sp=0xc00165bfe0 pc=0x2f3ecc1\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 502\n\t/go/pkg/mod/golang.org/x/sync@v0.20.0/errgroup/errgroup.go:78 +0x95\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2eb5e2e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffcb9d27d24\nrbp    0x7ffcb9d27cd8\nrsp    0x7ffcb9d27c90\nr8     0x0\nr9     0x0\nr10    0x393\nr11    0x246\nr12    0x7ffcb9d27d68\nr13    0x1\nr14    0xc9b9480\nr15    0x3\nrip    0x2eb5e2e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"c57dd962ef391925d5c6b7fb0887a08b2c528a5a1bbe4aa8835d8b4cd94b73ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"2297825ff7c934f1a0949564961ae25dba26cc537022d08ba17556c8f0143842","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bf5a40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4n5wv7, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-4n5wv7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4n5wv7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bf5a40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4n5wv7, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-4n5wv7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-4n5wv7, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"c9721c7d6308bc71d3ee0b1f2319451e855d7735b41174a9206ee76e472362b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"8f71bc7117807f930e17f3b000d46ab0054b7b803aafe1b4ee8186c3546bd095","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011a26c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-z9kl8k, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z9kl8k\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-z9kl8k, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011a26c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-z9kl8k, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-z9kl8k\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-z9kl8k, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"cb94ca0d6959b812c3aaeeb0e3e52e1fe7ed1dc87c2c796d1e3456094c535476","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"68cb4ec713f1e549306c75d11922bfcf13e13c6a72d65cec267dad29412315c3","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009e00c0\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-qx9jfx, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-qx9jfx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-qx9jfx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009e00c0\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-qx9jfx, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-qx9jfx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-qx9jfx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"cd2ebfcb981bba1d3c4bdb9fcdd9f3e872868c0acad71d96163a8b9145472518","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"0554e6ab0e6a614036a6bcbbbd203e4f4bb3b25ca9dcd4d9ab0f726bda20b283","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-sh5qkl/cluster-candidate-5-0-4glj7t should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012f0090\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-4glj7t, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4glj7t in resource group rg-candidate-5-0-sh5qkl, error: failed waiting for cluster=\"cluster-candidate-5-0-4glj7t\" in resourcegroup=\"rg-candidate-5-0-sh5qkl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4glj7t in resource group rg-candidate-5-0-sh5qkl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-sh5qkl/cluster-candidate-5-0-4glj7t should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012f0090\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-4glj7t, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4glj7t in resource group rg-candidate-5-0-sh5qkl, error: failed waiting for cluster=\"cluster-candidate-5-0-4glj7t\" in resourcegroup=\"rg-candidate-5-0-sh5qkl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4glj7t in resource group rg-candidate-5-0-sh5qkl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"cf471262a7db0a7a61ad4e2b0861d9bdc258c3ea6d005e0b4228ebbd78bc71bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"39a10aab7e47fd4e1832359e1be722185297231ff6b9d34348c591995b419911","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001012d20\u003e: \n    failed to create HCP cluster np-dg-minor-hzlq6d, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-dg-minor-hzlq6d in resource group rg-np-dg-minor-hzlq6d-cq92gl, error: failed waiting for cluster=\"np-dg-minor-hzlq6d\" in resourcegroup=\"rg-np-dg-minor-hzlq6d-cq92gl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-dg-minor-hzlq6d in resource group rg-np-dg-minor-hzlq6d-cq92gl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001012d20\u003e: \n    failed to create HCP cluster np-dg-minor-hzlq6d, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-dg-minor-hzlq6d in resource group rg-np-dg-minor-hzlq6d-cq92gl, error: failed waiting for cluster=\"np-dg-minor-hzlq6d\" in resourcegroup=\"rg-np-dg-minor-hzlq6d-cq92gl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-dg-minor-hzlq6d in resource group rg-np-dg-minor-hzlq6d-cq92gl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"dbf87d34c49390d552d4ea3d42947798feaf543750caa301f8eb731ba83805dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"438e6e3b40fc91b239b897d30af29235d11b1c36777b26c6539b8ce1e59c2bf2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001284060\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xj2l8r\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-xj2l8r, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001284060\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xj2l8r\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-xj2l8r, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"dd0028f39b2876a8930ff009139246d24cd3735bd1aff0ba400f5037187235c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes"}},{"row_id":"dec81fd8fefa2b9fb18a72ce84faee42d4adbd21a0ec53a776163599146dd9de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"4d09d877cb0ecc01ce0e481f52391481a408a35ec842dd40a7ebe3be0bdedbf2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-746ctc\" with version 4.21.0-rc.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00099f380\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-746ctc, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-746ctc in resource group rg-zstream-upgrade-4-21-lrhcfn, error: failed waiting for cluster=\"cluster-zstream-4-21-746ctc\" in resourcegroup=\"rg-zstream-upgrade-4-21-lrhcfn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-746ctc in resource group rg-zstream-upgrade-4-21-lrhcfn, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-746ctc\" with version 4.21.0-rc.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00099f380\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-746ctc, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-746ctc in resource group rg-zstream-upgrade-4-21-lrhcfn, error: failed waiting for cluster=\"cluster-zstream-4-21-746ctc\" in resourcegroup=\"rg-zstream-upgrade-4-21-lrhcfn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-746ctc in resource group rg-zstream-upgrade-4-21-lrhcfn, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"e36e0b55a66d4562c491e8026db6e05f33bfe9967f210ca47653e96e6070841d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"f915e6b02586f8161979be107e9090aa6286ef6a76578157f3f92ddd8ab7f30b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-nzrzjc/cluster-candidate-4-20-m67tzn should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009af3b0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-nzrzjc/cluster-candidate-4-20-m67tzn should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009af3b0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"e53557e7c7feca89f96c5e62025dc09f57ad3230e5b86275165594fb91edb61f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"1dbe28aaa1123f83802f1de707b5da7fc2853f3767e0a95ed577eaed89787464","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:150]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001c2d2c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n6wr5x, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-n6wr5x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n6wr5x, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:150]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001c2d2c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n6wr5x, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-n6wr5x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-n6wr5x, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"e5941777f7be124f573ea3209ebf3f814f48d99c05702f58ce5222ace34f9f5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"9f2ac729fe9d976da65402072c5ce163aec86e2fc9dd555a29c79a9026760bcb","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010108a0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l45rx5, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-l45rx5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l45rx5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010108a0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l45rx5, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-l45rx5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l45rx5, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"e9924f039cf597dc762bd3203a6eef0229cff8fe511a8e5794729bafd77b1ed4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"5133a9a3510c736dff1ba567230f495eb4fc871d89503d306c44984e205946f9","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c105a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-hrn49q, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-hrn49q\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-hrn49q, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c105a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-hrn49q, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-hrn49q\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-hrn49q, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"ea42b63a0f6acbff3f0cfeaae11a1f840bdb2d9f49376134ca21982913fc38c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"2d8d96f15913cd8c77c68593081a75d4522169b27e23a89f5029858d7d3efd91","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-zlg882/cluster-candidate-4-22-grxs7p should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a39a40\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-grxs7p, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-grxs7p in resource group rg-candidate-4-22-zlg882, error: failed waiting for cluster=\"cluster-candidate-4-22-grxs7p\" in resourcegroup=\"rg-candidate-4-22-zlg882\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-grxs7p in resource group rg-candidate-4-22-zlg882, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-zlg882/cluster-candidate-4-22-grxs7p should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a39a40\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-grxs7p, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-grxs7p in resource group rg-candidate-4-22-zlg882, error: failed waiting for cluster=\"cluster-candidate-4-22-grxs7p\" in resourcegroup=\"rg-candidate-4-22-zlg882\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-grxs7p in resource group rg-candidate-4-22-zlg882, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"eadf24c41786a9ddfa0ef7da09d53f94cb12baa19db114dfc040707ac63fe1b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"d1534c2f02127482bc57486a7ebe25aa656a4bb7299fc9040490863de324219b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-9tgpd2 with version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d14bd0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-9tgpd2, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9tgpd2 in resource group rg-np-version-upgrade-9tgpd2-dk4zzz, error: failed waiting for cluster=\"np-version-upgrade-cluster-9tgpd2\" in resourcegroup=\"rg-np-version-upgrade-9tgpd2-dk4zzz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9tgpd2 in resource group rg-np-version-upgrade-9tgpd2-dk4zzz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-9tgpd2 with version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d14bd0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-9tgpd2, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9tgpd2 in resource group rg-np-version-upgrade-9tgpd2-dk4zzz, error: failed waiting for cluster=\"np-version-upgrade-cluster-9tgpd2\" in resourcegroup=\"rg-np-version-upgrade-9tgpd2-dk4zzz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9tgpd2 in resource group rg-np-version-upgrade-9tgpd2-dk4zzz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"f9f671ec29b216c305f720ea4a596b60de57a2e0cc32be3bd502479226833abc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"7e24a60a3aaaaeec020d6593623c0904a5a964308294927373b3eb42b40b158b","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017815f0\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-h4fr85, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-h4fr85\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-h4fr85, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017815f0\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-h4fr85, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-h4fr85\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-h4fr85, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"fad6821424901212b339d6c73b499d3fb1da0d77cf0792af06c2f773a349f4ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"afb4b21aad6687d1326697d4b7a71abb51ae46583ac23b65d65e061a3fff86fe","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a5b60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-5q8rr7, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-5q8rr7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-5q8rr7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a5b60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-5q8rr7, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-5q8rr7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-5q8rr7, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"fb6700e88781a0ffcbf9bc9fd09880aa6adf3c3deaff49c0ffa97e30b26f89ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074860029835481088","occurred_at":"2026-07-08T14:15:53Z","failed_at":"e2e","signature_id":"15bbf6f42543fdf7cfcb73fae3e921ede323286b5e327c79169ed8f66e8f4119","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-6ggdq8\" with version 4.19.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00152a840\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-6ggdq8, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6ggdq8 in resource group rg-zstream-upgrade-4-19-hp6xzz, error: failed waiting for cluster=\"cluster-zstream-4-19-6ggdq8\" in resourcegroup=\"rg-zstream-upgrade-4-19-hp6xzz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6ggdq8 in resource group rg-zstream-upgrade-4-19-hp6xzz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-6ggdq8\" with version 4.19.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00152a840\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-6ggdq8, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6ggdq8 in resource group rg-zstream-upgrade-4-19-hp6xzz, error: failed waiting for cluster=\"cluster-zstream-4-19-6ggdq8\" in resourcegroup=\"rg-zstream-upgrade-4-19-hp6xzz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6ggdq8 in resource group rg-zstream-upgrade-4-19-hp6xzz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":44,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","79232f7fe5f1a65f3af30b7ba05ac2a1643a1a699ce682f3bc11178c03f0cbf4","a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41"],"matched_occurrences":44,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-08T09:41:32Z"},"failed_at":["e2e"],"failed_test_count":18,"occurrences":[{"row_id":"04a8fe1d703b3802bd5562dcc310b0c06cb2fd8c2aab6f4adcebe585b49ea809","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"853593d09741e209fe5dec6b4d039fc5839ccc8176bd1fb4cf6294a635dae0e6","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ce2540\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-nlbvcv, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-nlbvcv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-nlbvcv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ce2540\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-nlbvcv, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-nlbvcv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-nlbvcv, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"1b2ea50523deedbef8bf1d0b43617b6bbc277954335311a336a44d57535da896","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"2251af67413ee13c4c5e368b0277dd29224009821a309b46277adf7d0691d92a","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001292d50\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cx-rg-hcp-cluster in resource group cx-rg-cluster-g9rbzv, error: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-g9rbzv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cx-rg-hcp-cluster in resource group cx-rg-cluster-g9rbzv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001292d50\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cx-rg-hcp-cluster in resource group cx-rg-cluster-g9rbzv, error: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-g9rbzv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cx-rg-hcp-cluster in resource group cx-rg-cluster-g9rbzv, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"1e2f0917e58f78714357d9a80636cdc37fc6b98e8ceb5fb08fb7a881179614a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"c39b6ee185c377797cc31e99dd8d5b2d1c19a4bf52465ead53e538125a379ca6","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013bb500\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-9l5sfk, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-9l5sfk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-9l5sfk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013bb500\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-9l5sfk, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-9l5sfk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-9l5sfk, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"2232f0eef24b5bb00738c4c12970be310b51304ec3fdd47c535601dd7cb12247","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"9e84f1efe606a19589ecd84d8735e48bdf3cae74d15d58520ee2fa46791d6ad0","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001334510\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-bdxbbx, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-bdxbbx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-bdxbbx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001334510\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-bdxbbx, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-bdxbbx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-bdxbbx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"2ddb15a66918bc66d281ad36f7b4bc31e89b48fe309d3a85d9b9ceff2c77a2fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"f10682ffc40f91f35f9475cee00f4e30cdbfbdecc512a24a5d8f190926e14171","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0025850e0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-cqzk85\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cqzk85, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0025850e0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-cqzk85\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cqzk85, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-cqzk85\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cqzk85, error: context deadline exceeded"}},{"row_id":"30a65d1d7b965062cec4cd969f866ecc77c7d3e758f0991293a0b47b7cfcbd4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"a9d80ca86d6c087aef0d0dc940b389dc6271865d2787e0f0b8354ec151a74212","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:607]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00239f020\u003e: \n    failed to create NodePool npdg-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npdg-4-21 in resource group rg-np-downgrade-845955-mgp9lz, error: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-845955\" in resourcegroup=\"rg-np-downgrade-845955-mgp9lz\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:607]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00239f020\u003e: \n    failed to create NodePool npdg-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npdg-4-21 in resource group rg-np-downgrade-845955-mgp9lz, error: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-845955\" in resourcegroup=\"rg-np-downgrade-845955-mgp9lz\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-p92jfl, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p92jfl\" to finish creating: context deadline exceeded"}},{"row_id":"4746f46cc7c0f2f27b627a7a7da8c54ca79129a13769b7a9dc9adf828de0636a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"6662436447f7c11e5d01b1451bc685b4b1823ce737043dbbdc68cda120e256cf","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c6f560\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-xqz7hq, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xqz7hq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-xqz7hq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c6f560\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-xqz7hq, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-xqz7hq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-xqz7hq, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"6dab59d39da675c96b4f7a0c4d29d7e4eabcac53e658e585ba2bc3f93406d40e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"240a061aeaa5d44d7837cb663804ed0d248d2a6adedea28dc723c04ef4e1326e","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc000b267e0\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node e9n0y8t3z6s7y7t-np-1-6l2x9-pkqwh: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-h4zck/default: serviceaccount \"default\" not found\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc000b267e0\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node e9n0y8t3z6s7y7t-np-1-6l2x9-pkqwh: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-h4zck/default: serviceaccount \"default\" not found\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c06c4634c5f33bac64ee9a28a4387236f0ddb12cdeaa39ff33d99a21963b5e3f","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to create FIPS check pod: pod \"\u003cpod\u003e\" is forbidden: error looking up service account \u003cnamespace\u003e/default: serviceaccount \"default\" not found","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node e9n0y8t3z6s7y7t-np-1-6l2x9-pkqwh: failed to create FIPS check pod: pods \"fips-check-\" is forbidden: error looking up service account e2e-fips-test-h4zck/default: serviceaccount \"default\" not found"}},{"row_id":"88c39e3a2af8ac12943a691605e0aef8a41602a11c03d1d7570d2c2a9af35f31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"ee0a3501c1a0d5e64f3e9950f8f901c979e4a1e3b15484eeeca1022d09da8d6e","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:153]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00244fe30\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-p92jfl, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p92jfl\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:153]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00244fe30\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-p92jfl, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p92jfl\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-p92jfl, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p92jfl\" to finish creating: context deadline exceeded"}},{"row_id":"9385036db58ebfdbb2a482af139defc77e678ce20fc702475d62f7c987ca4155","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"c7dd31532f3fb1df869fdf3125cd542623e96eb4c34228eb7234b23f46929991","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0016da1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-cfqmg9\" in resourcegroup=\"rg-candidate-4-21-5l8mwr\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0016da1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-cfqmg9\" in resourcegroup=\"rg-candidate-4-21-5l8mwr\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-cfqmg9\" in resourcegroup=\"rg-candidate-4-21-5l8mwr\" to finish deleting: context canceled"}},{"row_id":"a0f322d46d287a49104d8d03bd003f9facf015376ccf09428fb4a1f7f39c4593","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"42a89d117274e2a1b0c87982e6b4f88a7f679aa71d7faa4ff76fa840202f81fc","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f19d40\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jbrspk\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:194]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f19d40\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jbrspk\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-jbrspk\" to finish creating: context deadline exceeded"}},{"row_id":"a34eeb3f27f94cf7002a9ee9b95537a653cb6c8c980a4874cbd8b9358493dfac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"992e661009bd6cc777dff7be4b9f18543ed54486ef1878bb220c1a3eeae9de6a","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:754]: failed to create nodepool npdg-421-419\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015aacf0\u003e: \n    failed to create NodePool npdg-421-419, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npdg-421-419 in resource group rg-np-dg-minor-2k4w8q-d2pcfx, error: failed waiting for nodepool=\"npdg-421-419\" for cluster \"np-dg-minor-2k4w8q\" in resourcegroup=\"rg-np-dg-minor-2k4w8q-d2pcfx\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:754]: failed to create nodepool npdg-421-419\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015aacf0\u003e: \n    failed to create NodePool npdg-421-419, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npdg-421-419 in resource group rg-np-dg-minor-2k4w8q-d2pcfx, error: failed waiting for nodepool=\"npdg-421-419\" for cluster \"np-dg-minor-2k4w8q\" in resourcegroup=\"rg-np-dg-minor-2k4w8q-d2pcfx\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-p92jfl, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p92jfl\" to finish creating: context deadline exceeded"}},{"row_id":"ba3c98fbc3dbb7a805e199275135832516ffb0aac9395574099161bfd66da962","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"a3b7a0d2c3e838c32f8c3c0a0c703c998a7038644320c2a1807d8b3f012cff23","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: failed to get admin REST config for cluster np-version-upgrade-cluster-gv255d\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002418720\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-gv255d\" in resourcegroup=\"rg-np-version-upgrade-gv255d-bzslj8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-gv255d in resource group rg-np-version-upgrade-gv255d-bzslj8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: failed to get admin REST config for cluster np-version-upgrade-cluster-gv255d\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002418720\u003e: \n    failed waiting for hcpCluster=\"np-version-upgrade-cluster-gv255d\" in resourcegroup=\"rg-np-version-upgrade-gv255d-bzslj8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-gv255d in resource group rg-np-version-upgrade-gv255d-bzslj8, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"np-version-upgrade-cluster-gv255d\" in resourcegroup=\"rg-np-version-upgrade-gv255d-bzslj8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster np-version-upgrade-cluster-gv255d in resource group rg-np-version-upgrade-gv255d-bzslj8, error: context deadline exceeded"}},{"row_id":"e44c3cf5022431f6124052ede9fb239eaba806665bb38089141bdc38d5662e85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"660c2d2ef27db1386f6fb6a6518159647b9c2f13e1b68a6c879e6ada6f83ab2d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-qxgd9w with version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012a89c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qxgd9w, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qxgd9w in resource group rg-np-version-upgrade-qxgd9w-lqr7rr, error: failed waiting for cluster=\"np-version-upgrade-cluster-qxgd9w\" in resourcegroup=\"rg-np-version-upgrade-qxgd9w-lqr7rr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qxgd9w in resource group rg-np-version-upgrade-qxgd9w-lqr7rr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-qxgd9w with version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012a89c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qxgd9w, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qxgd9w in resource group rg-np-version-upgrade-qxgd9w-lqr7rr, error: failed waiting for cluster=\"np-version-upgrade-cluster-qxgd9w\" in resourcegroup=\"rg-np-version-upgrade-qxgd9w-lqr7rr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-qxgd9w in resource group rg-np-version-upgrade-qxgd9w-lqr7rr, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"e4b29cd3be23f0d9ba7086c9f0590a3e01fd94fb9881069f7dc68e8b944646dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"9e3ad48475bba3ac4b5df6b42687239fbe9ee4e76afdd8a7ba5e5b0c0c72354f","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:477]: failed to create nodepool nps-420-422 at version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0024b1110\u003e: \n    failed to create NodePool nps-420-422, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nps-420-422 in resource group rg-np-skip-minor-lfkkhh-5f9msh, error: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-lfkkhh\" in resourcegroup=\"rg-np-skip-minor-lfkkhh-5f9msh\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:477]: failed to create nodepool nps-420-422 at version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0024b1110\u003e: \n    failed to create NodePool nps-420-422, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nps-420-422 in resource group rg-np-skip-minor-lfkkhh-5f9msh, error: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-lfkkhh\" in resourcegroup=\"rg-np-skip-minor-lfkkhh-5f9msh\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-p92jfl, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p92jfl\" to finish creating: context deadline exceeded"}},{"row_id":"f1bec4042d67c69b2d5cfcab7dc2f67089bb50fc05a8c2d809a28f496917b2a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"8f7f52b33bfba720232f67996e6746df05c55e1bf868cc452089392898d4dbdd","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e73890\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-7hwljb, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7hwljb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-7hwljb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e73890\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-7hwljb, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7hwljb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-7hwljb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-m67tzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: failed waiting for cluster=\"cluster-candidate-4-20-m67tzn\" in resourcegroup=\"rg-candidate-4-20-nzrzjc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-m67tzn in resource group rg-candidate-4-20-nzrzjc, error: context deadline exceeded"}},{"row_id":"f2b0f57dd9010768e72efe6a15b16f918ec4e8109482f84cb97e7255cb41f31c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"a5ae3383c025c2472192d42e693602e44ec17318c3c5c8d1ba5ddaa717587716","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:330]: failed to create nodepool npnoedge-4-20 at version 4.20.0\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014079e0\u003e: \n    failed to create NodePool npnoedge-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npnoedge-4-20 in resource group rg-np-noedge-j7lh88-lbpxgn, error: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-j7lh88\" in resourcegroup=\"rg-np-noedge-j7lh88-lbpxgn\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:330]: failed to create nodepool npnoedge-4-20 at version 4.20.0\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014079e0\u003e: \n    failed to create NodePool npnoedge-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npnoedge-4-20 in resource group rg-np-noedge-j7lh88-lbpxgn, error: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-j7lh88\" in resourcegroup=\"rg-np-noedge-j7lh88-lbpxgn\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-p92jfl, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p92jfl\" to finish creating: context deadline exceeded"}},{"row_id":"f75a280e00d2e12e8ba6d73ace84b0f0925fdcf1712b061ddcdb3efee66a8518","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2074790983152701440","occurred_at":"2026-07-08T09:41:32Z","failed_at":"e2e","signature_id":"abef8e61097f5ada28782f0e139e08fc161b405f6953ce3af4fe771e91e885d3","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.22\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000890ea0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-fr5x47\" resourcegroup=\"rg-np-version-upgrade-fr5x47-tjnxv8\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-fr5x47 in resource group rg-np-version-upgrade-fr5x47-tjnxv8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.22\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000890ea0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-fr5x47\" resourcegroup=\"rg-np-version-upgrade-fr5x47-tjnxv8\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-fr5x47 in resource group rg-np-version-upgrade-fr5x47-tjnxv8, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-cqzk85\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cqzk85, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","c06c4634c5f33bac64ee9a28a4387236f0ddb12cdeaa39ff33d99a21963b5e3f","f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2074674924374462464","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-08T02:00:22Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4d67b3235231ccc15930b691151d0daa76179ff43b57ff4ca919dc819a894d30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2074674924374462464","occurred_at":"2026-07-08T02:00:22Z","failed_at":"e2e","signature_id":"0b80769ddd1b37ce9bab26af28e6639d9ac29c535c4dfdaa44448a9369ea2146","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008eab80\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-c67zkg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-c67zkg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008eab80\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-c67zkg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-c67zkg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.fips-enabled\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.fips-enabled]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7aebc21de77cb64a5180833ea520c7de32315e635ee94e975fa51f1b8f3df2fd","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.fips-enabled\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ba92ee5d38ec0ec97996241540a9b1995b52af49b18ad852d9181080ae25e80a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2074674924374462464","occurred_at":"2026-07-08T02:00:22Z","failed_at":"e2e","signature_id":"de8314c475995e0785e91e95c202ce3fffed3fe3c3aef8af44fc920a56292561","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should reject an HCP cluster creation with invalid FIPS tag value","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ac9ea1609c23f13fc59d25bc0226d6ddbb91e9217b1f67c5624f53eb77e8333","failure_pattern":"error should indicate FIPS value must be true or false","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:203]: error should indicate FIPS value must be true or false"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ac9ea1609c23f13fc59d25bc0226d6ddbb91e9217b1f67c5624f53eb77e8333","7aebc21de77cb64a5180833ea520c7de32315e635ee94e975fa51f1b8f3df2fd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}