{"meta":{"date":"2026-04-09","timezone":"UTC","generated_at":"2026-08-30T09:41:05Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":9,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":4,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042388655787479040","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-09T23:46:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-09T23:42:00Z"},"failed_at":["e2e","unknown"],"failed_test_count":33,"occurrences":[{"row_id":"097efc37486edff66a1ceeb26b007d24e2589c27e57f9cd2fdef7b740ea127dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"20d80203cbad1be4b26371494dcc6cc544922c19a12c074a01d8bb2de59736ce","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f54d20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f54d20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"0f26f861084171e6896af3397021ae2828cca12b022fefc4d814f8b78eba9b50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"1434b8b1beae1e2462f904ed8c7fd391d5c039e029a420e6cf6d081b0c0d0b0a","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001736fc0\u003e: \n failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-qmrn9q, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qmrn9q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-qmrn9q, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001736fc0\u003e: \n failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-qmrn9q, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qmrn9q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-qmrn9q, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"1f2487f58c5df8b2554ecb532b69ba597ecce8bcd3d90bacb3d5b30bdb7ad509","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"51d263ce91b4409e4460d92f782dcbcf48a02d5d56f79d54b7af9e35ec33c3be","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0003805d0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gstdrp, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-gstdrp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gstdrp, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0003805d0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gstdrp, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-gstdrp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-gstdrp, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"25034009447b363a3cad90e659a1922d18e5a3f2ae0c263fd9305aa841dedca6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"1ff88c7cdd3499f0b611863b832cf641a1174675f55466b3a9b7339a91830ff1","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b62f90\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-mvd2qk, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-mvd2qk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-mvd2qk, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b62f90\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-mvd2qk, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-mvd2qk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-mvd2qk, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"27e8efd57073565c6b4672408d50f69847c59b057e5b88b386ea0915b058a355","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"e7b16355619287ee49663b13c0880b88572d6800c4fb83e0f33d39be07eaf924","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775785195 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to use workload identity via the cluster OIDC issuer URL [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:169\n \"ts\"=\"2026-04-10 01:39:55.131080\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-use-workload-identity-via-the-cluster-OIDC-issuer-URL|pid:903\"\n \"ts\"=\"2026-04-10 01:39:55.137582\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/10/26 01:39:55.137\n \"ts\"=\"2026-04-10 01:39:55.137651\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"oidc-wi-dnbtp6\"\n STEP: creating cluster parameters @ 04/10/26 01:39:59.261\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/10/26 01:39:59.261\n STEP: creating the HCP cluster @ 04/10/26 01:41:09.86\n \"ts\"=\"2026-04-10 01:41:09.861125\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"oidc-wi-cluster\" \"resourceGroup\"=\"oidc-wi-dnbtp6\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775785195 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to use workload identity via the cluster OIDC issuer URL [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:169\n \"ts\"=\"2026-04-10 01:39:55.131080\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-use-workload-identity-via-the-cluster-OIDC-issuer-URL|pid:903\"\n \"ts\"=\"2026-04-10 01:39:55.137582\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/10/26 01:39:55.137\n \"ts\"=\"2026-04-10 01:39:55.137651\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"oidc-wi-dnbtp6\"\n STEP: creating cluster parameters @ 04/10/26 01:39:59.261\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/10/26 01:39:59.261\n STEP: creating the HCP cluster @ 04/10/26 01:41:09.86\n \"ts\"=\"2026-04-10 01:41:09.861125\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"oidc-wi-cluster\" \"resourceGroup\"=\"oidc-wi-dnbtp6\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"395231134b3aeccdc687563d5e8ec617b4fde67f1c0d479be192ca1c33cc46b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"f04a5eed1e2f72f7a0d6b6ed6f9ad8509a8864946852b1905eb5300b43ce6b51","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012a5590\u003e: \n failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-49vb8b, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-49vb8b\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-49vb8b, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012a5590\u003e: \n failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-49vb8b, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-49vb8b\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-49vb8b, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"3c9204199264eb66ecbb68a9cc7fd8e7d428cc8e6c51950fe92603e24fdf92bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"66f5e30a0278bda3bbd019ac2f4b9bb0e570da30f3b70999fa97794a1ccf19be","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010fa720\u003e: \n failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-tk5vh5, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-tk5vh5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-tk5vh5, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010fa720\u003e: \n failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-tk5vh5, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-tk5vh5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-tk5vh5, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"3f913c42b175c6f7aac4e2b301680dfd79de5b916d28639b89cd5bd28e849d62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"d77696467a2d442aa450fda4be7fa109a0efae503aae9c34c88fb70b01c15bb6","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008c6930\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-8flt69, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-8flt69 in resource group rg-cp-ystream-upgrade-4-21-xqc5mj, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-8flt69\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-xqc5mj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-8flt69 in resource group rg-cp-ystream-upgrade-4-21-xqc5mj, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008c6930\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-8flt69, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-8flt69 in resource group rg-cp-ystream-upgrade-4-21-xqc5mj, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-8flt69\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-xqc5mj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-8flt69 in resource group rg-cp-ystream-upgrade-4-21-xqc5mj, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"405a083d0c156b108b011851ccfdbe1ac3511d89ad597abf817a915c3da5eb0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"763cc5ea46d3724878f97a7c2aeb317ebbf14cfa39e2f628b3b2132b6b57ae00","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011f2e10\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-48kzhd, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-48kzhd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-48kzhd, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011f2e10\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-48kzhd, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-48kzhd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-48kzhd, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"4140fbd17d5a712088c7d885798195f6f29afe98d7013d71d61fc0329ff0231e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"a44703a6832a34a86e069e863ce95031cec6241d092adc5005108861b3a236fc","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.995667\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.997695\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.998987\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:26.000957\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:26.003463\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:26.005539\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:26.007826\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:26.010162\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.012551\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.014656\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.016406\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.376864\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=118 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h57m2s\"\n \"ts\"=\"2026-04-10 01:43:28.576766\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=119 \"elapsed\"=\"1h58m3s\"\n STEP: creating resource group for admin credential lifecycle testing @ 04/10/26 01:43:28.576\n \"ts\"=\"2026-04-10 01:43:28.576865\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"admin-credential-lifecycle-test-r46lz5\"\n STEP: creating cluster parameters @ 04/10/26 01:43:32.738\n STEP: creating customer resources @ 04/10/26 01:43:32.738\n STEP: starting HCP cluster creation asynchronously @ 04/10/26 01:44:53.672\n \"ts\"=\"2026-04-10 01:44:53.672604\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"admin-cred-lifecycle-85ks84\" \"resourceGroup\"=\"admin-credential-lifecycle-test-r46lz5\"\n STEP: waiting for cluster to appear and testing admin credentials while in deploying state @ 04/10/26 01:44:54.783\n \"ts\"=\"2026-04-10 01:44:54.784022\" \"level\"=0 \"msg\"=\"creating cluster, waiting for it to reach a terminal state\"\n \"ts\"=\"2026-04-10 01:44:55.173149\" \"level\"=0 \"msg\"=\"Cluster provisioning state updated\" \"provisioningState\"=\"Accepted\"\n STEP: testing admin credentials while cluster is in deploying state @ 04/10/26 01:44:55.173\n STEP: verifying admin credentials request fails with HTTP 409 CONFLICT on deploying cluster @ 04/10/26 01:44:55.772\n \"ts\"=\"2026-04-10 01:44:55.772589\" \"level\"=0 \"msg\"=\"Admin credentials request correctly returned 409 conflict error while cluster is deploying\"\ntime=\"2026-04-10T01:45:22Z\" level=info msg=\"Received signal.\" signal=interrupt\nInterrupted, terminating testsCommand Error: signal: killed\nDeserializaion Error: no output from command\nfree identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.995667\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.997695\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.998987\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:26.000957\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:26.003463\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:26.005539\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:26.007826\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:26.010162\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.012551\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.014656\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.016406\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.376864\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=118 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h57m2s\"\n \"ts\"=\"2026-04-10 01:43:28.576766\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=119 \"elapsed\"=\"1h58m3s\"\n STEP: creating resource group for admin credential lifecycle testing @ 04/10/26 01:43:28.576\n \"ts\"=\"2026-04-10 01:43:28.576865\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"admin-credential-lifecycle-test-r46lz5\"\n STEP: creating cluster parameters @ 04/10/26 01:43:32.738\n STEP: creating customer resources @ 04/10/26 01:43:32.738\n STEP: starting HCP cluster creation asynchronously @ 04/10/26 01:44:53.672\n \"ts\"=\"2026-04-10 01:44:53.672604\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"admin-cred-lifecycle-85ks84\" \"resourceGroup\"=\"admin-credential-lifecycle-test-r46lz5\"\n STEP: waiting for cluster to appear and testing admin credentials while in deploying state @ 04/10/26 01:44:54.783\n \"ts\"=\"2026-04-10 01:44:54.784022\" \"level\"=0 \"msg\"=\"creating cluster, waiting for it to reach a terminal state\"\n \"ts\"=\"2026-04-10 01:44:55.173149\" \"level\"=0 \"msg\"=\"Cluster provisioning state updated\" \"provisioningState\"=\"Accepted\"\n STEP: testing admin credentials while cluster is in deploying state @ 04/10/26 01:44:55.173\n STEP: verifying admin credentials request fails with HTTP 409 CONFLICT on deploying cluster @ 04/10/26 01:44:55.772\n \"ts\"=\"2026-04-10 01:44:55.772589\" \"level\"=0 \"msg\"=\"Admin credentials request correctly returned 409 conflict error while cluster is deploying\"\ntime=\"2026-04-10T01:45:22Z\" level=info msg=\"Received signal.\" signal=interrupt\nInterrupted, terminating testsCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"41954f4e1abbcd14d36c3a991bdd9e109ad54a6e7d0f04b51d3e88c00f4b99dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"4bd17e5262806660ea3cb7307dacae443f38c55f69a35963fbf5b93d093d7077","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001887080\u003e: \n failed to create HCP cluster cluster-zstream-4-19-pjnf9v, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pjnf9v in resource group rg-zstream-upgrade-4-19-vwgn9q, error: failed waiting for cluster=\"cluster-zstream-4-19-pjnf9v\" in resourcegroup=\"rg-zstream-upgrade-4-19-vwgn9q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pjnf9v in resource group rg-zstream-upgrade-4-19-vwgn9q, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001887080\u003e: \n failed to create HCP cluster cluster-zstream-4-19-pjnf9v, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pjnf9v in resource group rg-zstream-upgrade-4-19-vwgn9q, error: failed waiting for cluster=\"cluster-zstream-4-19-pjnf9v\" in resourcegroup=\"rg-zstream-upgrade-4-19-vwgn9q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-pjnf9v in resource group rg-zstream-upgrade-4-19-vwgn9q, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"42ffa7fd2dc533bdfabb18562a8168cecd3e75be46d94da2a6f17d045e5aefc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"f4da3c4b9ddd4ac2311fdb82b1a39837f9b3df6336381691c7f2b8bed4807b49","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775785253 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n \"ts\"=\"2026-04-10 01:40:53.520546\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:917\"\n \"ts\"=\"2026-04-10 01:40:53.527501\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/10/26 01:40:53.527\n \"ts\"=\"2026-04-10 01:40:53.527553\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-fj8dkb\"\n STEP: creating cluster parameters @ 04/10/26 01:40:57.823\n STEP: creating customer resources @ 04/10/26 01:40:57.823\n STEP: creating the HCP cluster @ 04/10/26 01:42:18.762\n \"ts\"=\"2026-04-10 01:42:18.762988\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-fj8dkb\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775785253 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n \"ts\"=\"2026-04-10 01:40:53.520546\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:917\"\n \"ts\"=\"2026-04-10 01:40:53.527501\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/10/26 01:40:53.527\n \"ts\"=\"2026-04-10 01:40:53.527553\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-fj8dkb\"\n STEP: creating cluster parameters @ 04/10/26 01:40:57.823\n STEP: creating customer resources @ 04/10/26 01:40:57.823\n STEP: creating the HCP cluster @ 04/10/26 01:42:18.762\n \"ts\"=\"2026-04-10 01:42:18.762988\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-fj8dkb\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"49221e30bbdd8e5d668c7048797c0a329ed9f3ab1e2268b44d6ea80a55858a70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"ac3b4a72760a724c1b769b45f8e59a39b612336b94a6c45e7386811540975018","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009527b0\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5fkhxr, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5fkhxr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5fkhxr, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009527b0\u003e: \n failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5fkhxr, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5fkhxr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5fkhxr, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"4e511edb384caeac618de796498ab5c229ec374beb16cb4f61f9c19d3d25c59f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"947d109d8d43c9f5ce0a3ad65337cbc76c5e5f47e6a6e8e25fe216f7aba3aba0","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"signed\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.970126\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.972003\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.974055\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.976115\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.978443\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.980652\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.983036\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.985005\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.986981\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.988915\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.990940\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.992341\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:25.994468\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:25.995916\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:25.999219\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.001652\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.362711\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating a resource group @ 04/10/26 01:42:27.362\n \"ts\"=\"2026-04-10 01:42:27.362860\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"tls-endpoint-cluster-k5wl7j\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/10/26 01:42:31.242\n STEP: creating a standard hcp cluster @ 04/10/26 01:43:33.26\n \"ts\"=\"2026-04-10 01:43:33.260551\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"tls-endpoint-hcp-cluster\" \"resourceGroup\"=\"tls-endpoint-cluster-k5wl7j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nsigned\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.970126\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.972003\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.974055\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.976115\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.978443\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.980652\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.983036\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.985005\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.986981\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.988915\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.990940\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.992341\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:25.994468\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:25.995916\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:25.999219\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.001652\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.362711\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating a resource group @ 04/10/26 01:42:27.362\n \"ts\"=\"2026-04-10 01:42:27.362860\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"tls-endpoint-cluster-k5wl7j\"\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/10/26 01:42:31.242\n STEP: creating a standard hcp cluster @ 04/10/26 01:43:33.26\n \"ts\"=\"2026-04-10 01:43:33.260551\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"tls-endpoint-hcp-cluster\" \"resourceGroup\"=\"tls-endpoint-cluster-k5wl7j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"520b0a6f2e05d06a5cae758e44192eb4051cc3d10cb19461ffdd3047ca5e889c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"330cb64c3c554ccfae42debfd0d76fd8797f0c0fcaa880bdd959f65399919c88","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008f6090\u003e: \n failed to create HCP cluster cluster-zstream-4-20-27sf7p, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-27sf7p in resource group rg-zstream-upgrade-4-20-jh4tp7, error: failed waiting for cluster=\"cluster-zstream-4-20-27sf7p\" in resourcegroup=\"rg-zstream-upgrade-4-20-jh4tp7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-27sf7p in resource group rg-zstream-upgrade-4-20-jh4tp7, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008f6090\u003e: \n failed to create HCP cluster cluster-zstream-4-20-27sf7p, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-27sf7p in resource group rg-zstream-upgrade-4-20-jh4tp7, error: failed waiting for cluster=\"cluster-zstream-4-20-27sf7p\" in resourcegroup=\"rg-zstream-upgrade-4-20-jh4tp7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-27sf7p in resource group rg-zstream-upgrade-4-20-jh4tp7, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"6ba2908d05a3f9db7b7c810ae43c64e0f03cdc88d225d46bd87fb844ef0ee134","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"a3601dc891786f1e347882a854cda79072206de659df846624d91488873d6788","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012da690\u003e: \n failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-r8p9n9, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-r8p9n9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-r8p9n9, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012da690\u003e: \n failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-r8p9n9, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-r8p9n9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-r8p9n9, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"6ca0a0a70e689f033bafa1ddd08cacc8a0820c9f23715c71f37753e016b6da14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"b8be7be9b69c391c1558c0d3fba34bce9063369ea28ca2c53d29ce2a33a4fdfc","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000960ea0\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-tsvtqx, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-tsvtqx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-tsvtqx, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000960ea0\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-tsvtqx, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-tsvtqx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-tsvtqx, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"786c7df6295f40d96c690f68356fe6a6255915c5e3e78a3056026ed967d2e62c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"144724c9be25fac726ad599942218985e7cb1c767a58c2aaf40d3e1a56b3670d","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775785307 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n \"ts\"=\"2026-04-10 01:41:47.102900\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-not-be-able-to-perform-various-invalid-operations-on-cluster-resources|pid:930\"\n \"ts\"=\"2026-04-10 01:41:49.288805\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"2s\"\n STEP: creating a resource group @ 04/10/26 01:41:49.288\n \"ts\"=\"2026-04-10 01:41:49.288894\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-6b9z9j\"\n STEP: creating cluster parameters @ 04/10/26 01:41:54.08\n STEP: creating customer resources @ 04/10/26 01:41:54.081\n STEP: creating the HCP cluster @ 04/10/26 01:43:05.737\n \"ts\"=\"2026-04-10 01:43:05.737229\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-6b9z9j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775785307 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n \"ts\"=\"2026-04-10 01:41:47.102900\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-not-be-able-to-perform-various-invalid-operations-on-cluster-resources|pid:930\"\n \"ts\"=\"2026-04-10 01:41:49.288805\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"2s\"\n STEP: creating a resource group @ 04/10/26 01:41:49.288\n \"ts\"=\"2026-04-10 01:41:49.288894\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-6b9z9j\"\n STEP: creating cluster parameters @ 04/10/26 01:41:54.08\n STEP: creating customer resources @ 04/10/26 01:41:54.081\n STEP: creating the HCP cluster @ 04/10/26 01:43:05.737\n \"ts\"=\"2026-04-10 01:43:05.737229\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-6b9z9j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"8e304af3cb61f87950ef0b7afddf257fb55ac4ec38151da0aa1569f426a9dc63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T01:45:22Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"8e87948cbeb40fdfa8388896197b66021954725c063af372a590190e05f98822","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"5143f678069366b9c72b55d545d3d5ed863e67f5752c369766a1f2194e58b299","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f6ecc0\u003e: \n failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-wdgc8m, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdgc8m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-wdgc8m, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f6ecc0\u003e: \n failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-wdgc8m, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdgc8m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-wdgc8m, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"92c845acfd2fd5e74bb6e8115244c8d235b38064c5ce3e1e7257b94ff58316a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"e777e665a1ca73ae9f55c94dd841945efb9238d4af249f7d2ae35961c1bdc2c5","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001487230\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-pz5zl4, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pz5zl4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-pz5zl4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001487230\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-pz5zl4, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pz5zl4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-pz5zl4, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"a41893943dd23c4dfd9afed0a10351c55812dc2d180ae39aac9660ea8bc349d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"9fa9e212bfc5e515204db2c8cda1c2e9df00f18528747dd58502103dd58209f9","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00180eba0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-wlj52v, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-wlj52v in resource group rg-zstream-upgrade-4-21-pzzmrb, error: failed waiting for cluster=\"cluster-zstream-4-21-wlj52v\" in resourcegroup=\"rg-zstream-upgrade-4-21-pzzmrb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-wlj52v in resource group rg-zstream-upgrade-4-21-pzzmrb, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00180eba0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-wlj52v, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-wlj52v in resource group rg-zstream-upgrade-4-21-pzzmrb, error: failed waiting for cluster=\"cluster-zstream-4-21-wlj52v\" in resourcegroup=\"rg-zstream-upgrade-4-21-pzzmrb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-wlj52v in resource group rg-zstream-upgrade-4-21-pzzmrb, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"a614856dd631be5f9b3a87557aef67d429efe1ef6add511e169ee74c339beca1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"d26642690096c377827f89a1b690bbd6e91c6a40cad085c73f784a6bea3470e5","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775778325 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group [PreLaunchSetup:None, ARO-HCP-RP-API-Compatible, Importance:Critical, Positivity:Positive]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35\n \"ts\"=\"2026-04-09 23:45:25.749409\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=3 \"specID\"=\"Customer-should-be-able-to-create-several-HCP-clusters-in-their-customer-resource-group,-but-not-in-the-same-managed-resource-group|pid:120\"\n \"ts\"=\"2026-04-09 23:45:25.773221\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=3 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a shared customer resource group @ 04/09/26 23:45:25.773\n \"ts\"=\"2026-04-09 23:45:25.773310\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"customer-rg-xmtqnl\"\n STEP: creating customer infrastructure and managed identities for first cluster @ 04/09/26 23:45:29.579\n STEP: creating customer infrastructure and managed identities for second cluster @ 04/09/26 23:46:53.764\n STEP: starting creation of both clusters in parallel @ 04/09/26 23:48:04.669\n \"ts\"=\"2026-04-09 23:48:04.669788\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster\" \"resourceGroup\"=\"customer-rg-xmtqnl\"\n \"ts\"=\"2026-04-09 23:48:13.377207\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster2\" \"resourceGroup\"=\"customer-rg-xmtqnl\"\n STEP: waiting for first cluster to complete creation @ 04/09/26 23:48:26.383\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775778325 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group [PreLaunchSetup:None, ARO-HCP-RP-API-Compatible, Importance:Critical, Positivity:Positive]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35\n \"ts\"=\"2026-04-09 23:45:25.749409\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=3 \"specID\"=\"Customer-should-be-able-to-create-several-HCP-clusters-in-their-customer-resource-group,-but-not-in-the-same-managed-resource-group|pid:120\"\n \"ts\"=\"2026-04-09 23:45:25.773221\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=3 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a shared customer resource group @ 04/09/26 23:45:25.773\n \"ts\"=\"2026-04-09 23:45:25.773310\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"customer-rg-xmtqnl\"\n STEP: creating customer infrastructure and managed identities for first cluster @ 04/09/26 23:45:29.579\n STEP: creating customer infrastructure and managed identities for second cluster @ 04/09/26 23:46:53.764\n STEP: starting creation of both clusters in parallel @ 04/09/26 23:48:04.669\n \"ts\"=\"2026-04-09 23:48:04.669788\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster\" \"resourceGroup\"=\"customer-rg-xmtqnl\"\n \"ts\"=\"2026-04-09 23:48:13.377207\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster2\" \"resourceGroup\"=\"customer-rg-xmtqnl\"\n STEP: waiting for first cluster to complete creation @ 04/09/26 23:48:26.383\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"a8934216c5b02b1645ff4b0de38f8f964cf07b5d28c6b7ae11d9a22cb5a01b3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"bcf9a78e2f0d6aacfd2a03648c0125ea09a1d71a284597b72aa79e4ee078e01f","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"ners but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.994626\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.996713\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.999752\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:26.001403\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:26.003322\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:26.005122\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:26.007258\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:26.009086\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:26.011684\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:26.013517\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:26.015514\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:26.017919\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.020182\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.022344\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.024553\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.378309\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=118 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h57m2s\"\n \"ts\"=\"2026-04-10 01:43:28.584897\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=119 \"elapsed\"=\"1h58m3s\"\n STEP: creating a resource group @ 04/10/26 01:43:28.584\n \"ts\"=\"2026-04-10 01:43:28.584987\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"patch-tags-zgdlx7\"\n STEP: creating cluster parameters @ 04/10/26 01:43:32.587\n STEP: creating customer resources @ 04/10/26 01:43:32.587\n STEP: creating the HCP cluster @ 04/10/26 01:44:53.237\n \"ts\"=\"2026-04-10 01:44:53.238008\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"patch-tags-cluster\" \"resourceGroup\"=\"patch-tags-zgdlx7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nners but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.994626\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.996713\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.999752\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:26.001403\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:26.003322\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:26.005122\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:26.007258\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:26.009086\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:26.011684\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:26.013517\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:26.015514\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:26.017919\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.020182\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.022344\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.024553\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.378309\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=118 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h57m2s\"\n \"ts\"=\"2026-04-10 01:43:28.584897\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=119 \"elapsed\"=\"1h58m3s\"\n STEP: creating a resource group @ 04/10/26 01:43:28.584\n \"ts\"=\"2026-04-10 01:43:28.584987\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"patch-tags-zgdlx7\"\n STEP: creating cluster parameters @ 04/10/26 01:43:32.587\n STEP: creating customer resources @ 04/10/26 01:43:32.587\n STEP: creating the HCP cluster @ 04/10/26 01:44:53.237\n \"ts\"=\"2026-04-10 01:44:53.238008\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"patch-tags-cluster\" \"resourceGroup\"=\"patch-tags-zgdlx7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"aa2b1a14dda5dbda6f038cc4ef4d457daff74c51a13309221d43c069ca069462","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"25112cf607019a413ef4df94318c39f9c02def0919498039896d48d73186ae50","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"ity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.976307\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.978331\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.979738\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.982005\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.984519\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.986359\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.988639\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.990629\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.993262\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.995084\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.997308\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.999692\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:26.001244\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.003164\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.004509\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.006500\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.375412\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating a resource group @ 04/10/26 01:42:27.375\n \"ts\"=\"2026-04-10 01:42:27.375486\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"autoscaling-cluster-vtfl2j\"\n STEP: creating customer resources @ 04/10/26 01:42:31.654\n STEP: creating the cluster @ 04/10/26 01:43:40.574\n \"ts\"=\"2026-04-10 01:43:40.574885\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"autoscaling-hcp-cluster\" \"resourceGroup\"=\"autoscaling-cluster-vtfl2j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.976307\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.978331\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.979738\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.982005\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.984519\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.986359\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.988639\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.990629\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.993262\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.995084\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.997308\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.999692\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:26.001244\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.003164\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.004509\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.006500\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.375412\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating a resource group @ 04/10/26 01:42:27.375\n \"ts\"=\"2026-04-10 01:42:27.375486\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"autoscaling-cluster-vtfl2j\"\n STEP: creating customer resources @ 04/10/26 01:42:31.654\n STEP: creating the cluster @ 04/10/26 01:43:40.574\n \"ts\"=\"2026-04-10 01:43:40.574885\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"autoscaling-hcp-cluster\" \"resourceGroup\"=\"autoscaling-cluster-vtfl2j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"aa5a577600b796f247df156638f3773ee8296893b3bef9f5486bf539430fb048","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"ceabc22814b9276f1ff45e9ae3bafd075ad167466b20198195bf855b7b2d2c52","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001345860\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7hbmqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-7hbmqr, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001345860\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7hbmqr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-7hbmqr, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"b86aafcf36d94d5b2aa67eda0935011e6fd6b0fb98fa8921a612dafcf04f4f16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"d7722e9b5999c253e78f65c34ee8909e1f5f33ad52d970eaea64c9e567cf6c35","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011f0ed0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-gnrpbw, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-gnrpbw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-gnrpbw, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011f0ed0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-gnrpbw, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-gnrpbw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-gnrpbw, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"be9e30fd76e9ba17c25b6c2328ee319aae2b8e74c3cf84e3c4338f7ae305a04c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"2ee08db4e3b091585cb0d893f015ff959a35d3c08ed6f9c2c96d9d598749655e","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff67b0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-77fx42, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-77fx42 in resource group rg-np-version-upgrade-77fx42-prfhcc, error: failed waiting for cluster=\"np-version-upgrade-cluster-77fx42\" in resourcegroup=\"rg-np-version-upgrade-77fx42-prfhcc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-77fx42 in resource group rg-np-version-upgrade-77fx42-prfhcc, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff67b0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-77fx42, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-77fx42 in resource group rg-np-version-upgrade-77fx42-prfhcc, error: failed waiting for cluster=\"np-version-upgrade-cluster-77fx42\" in resourcegroup=\"rg-np-version-upgrade-77fx42-prfhcc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-77fx42 in resource group rg-np-version-upgrade-77fx42-prfhcc, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"ce45a1a8fe43fed2bbbf24f495829cbef5f94b4127a7b0aaa88a0479f1837c36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"e02eea0acb0857847efd731148c95427f368bed721e2406ad6b6e8de41fad190","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e950e0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-6gfpn9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-6gfpn9, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e950e0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-6gfpn9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-6gfpn9, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"dbb92c7216a4e11909233d77f189dbeee107337adb7cf1e0890ffae44c2d8498","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"cf618007959110b43c9382ce0265e98bf5e4ae17099efc8020ed34e576a1d184","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775782039 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:223\nW0410 00:47:19.461902 876 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 00:47:19.544986 876 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 00:47:19.545244\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:876\"\n \"ts\"=\"2026-04-10 00:47:19.551002\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 00:47:19.551\n \"ts\"=\"2026-04-10 00:47:19.551072\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 00:47:23.522\nW0410 00:47:23.605176 876 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/10/26 00:47:23.605\n STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 00:48:44.534\n \"ts\"=\"2026-04-10 00:48:44.534430\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-9s68vn\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:129 @ 04/10/26 01:33:44.539\n \"ts\"=\"2026-04-10 01:33:44.540116\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 01:33:44.540243\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-9s68vn-vkqvp9\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\n \"ts\"=\"2026-04-10 01:33:44.866362\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-9s68vn-vkqvp9\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 01:33:47.540364\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 01:33:47.540506\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 01:33:47.540574\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 01:33:47.541736\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\"\n \"ts\"=\"2026-04-10 01:33:47.541753\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\n \"ts\"=\"2026-04-10 01:33:47.921967\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\" \"deployment\"=\"pooledNonMsiScopedAssignments-gs3eml53zo4am\"\n \"ts\"=\"2026-04-10 01:33:48.225104\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-9s68vn-vkqvp9\"\n \"ts\"=\"2026-04-10 01:33:48.267881\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 01:33:48.835002\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"msi-deployment-148\"\n \"ts\"=\"2026-04-10 01:33:48.983763\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 01:33:48.983864\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 01:33:48.983978\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775782039 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:223\nW0410 00:47:19.461902 876 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 00:47:19.544986 876 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 00:47:19.545244\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:876\"\n \"ts\"=\"2026-04-10 00:47:19.551002\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 00:47:19.551\n \"ts\"=\"2026-04-10 00:47:19.551072\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 00:47:23.522\nW0410 00:47:23.605176 876 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/10/26 00:47:23.605\n STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 00:48:44.534\n \"ts\"=\"2026-04-10 00:48:44.534430\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-9s68vn\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:129 @ 04/10/26 01:33:44.539\n \"ts\"=\"2026-04-10 01:33:44.540116\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 01:33:44.540243\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-9s68vn-vkqvp9\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\n \"ts\"=\"2026-04-10 01:33:44.866362\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-9s68vn-vkqvp9\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 01:33:47.540364\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 01:33:47.540506\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 01:33:47.540574\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 01:33:47.541736\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\"\n \"ts\"=\"2026-04-10 01:33:47.541753\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\n \"ts\"=\"2026-04-10 01:33:47.921967\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\" \"deployment\"=\"pooledNonMsiScopedAssignments-gs3eml53zo4am\"\n \"ts\"=\"2026-04-10 01:33:48.225104\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-9s68vn-vkqvp9\"\n \"ts\"=\"2026-04-10 01:33:48.267881\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 01:33:48.835002\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"msi-deployment-148\"\n \"ts\"=\"2026-04-10 01:33:48.983763\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 01:33:48.983864\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 01:33:48.983978\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9s68vn-v74ng7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e651ee820c1ecbb9fc120d9e33b44bb34892fd8f62b1ad96bdb7e7824dd4943b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"e1c9b11aabb04b01cf379a5ddb36533a49ce1283f8171a5e595ace8671ce7942","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:116]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009a15f0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-5h55f6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5h55f6, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:116]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009a15f0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-5h55f6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5h55f6, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}},{"row_id":"f09c13c8267a6dbbdcfaa7bade14aa7f5713ae2043c60b8cc4876d3d84288136","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"9b8dfe83a7175b3c3ff1dfc846c5bcb0af067c5e163ffa5144f809ba04cb7fac","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"prod/parallel","failure_text":"ity containers but only 0 are assigned\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.973178\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.975182\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.977194\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.979493\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.981426\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.983561\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.985249\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.987325\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.989229\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.991218\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.993463\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.995876\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:25.997941\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.000365\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.002150\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.004219\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.369138\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating resource group @ 04/10/26 01:42:27.369\n \"ts\"=\"2026-04-10 01:42:27.369233\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-stable-4-20-c5n4s2\"\n STEP: creating customer resources @ 04/10/26 01:42:31.433\n STEP: creating the HCP cluster with version '4.20' on stable channel @ 04/10/26 01:43:32.288\n \"ts\"=\"2026-04-10 01:43:32.288564\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-stable-4-20-4vbjtk\" \"resourceGroup\"=\"rg-stable-4-20-c5n4s2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nity containers but only 0 are assigned\" \"attempt\"=101 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h40m0s\"\n \"ts\"=\"2026-04-10 01:26:25.973178\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n \"ts\"=\"2026-04-10 01:27:25.975182\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n \"ts\"=\"2026-04-10 01:28:25.977194\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n \"ts\"=\"2026-04-10 01:29:25.979493\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n \"ts\"=\"2026-04-10 01:30:25.981426\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n \"ts\"=\"2026-04-10 01:31:25.983561\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n \"ts\"=\"2026-04-10 01:32:25.985249\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n \"ts\"=\"2026-04-10 01:33:25.987325\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n \"ts\"=\"2026-04-10 01:34:25.989229\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n \"ts\"=\"2026-04-10 01:35:25.991218\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n \"ts\"=\"2026-04-10 01:36:25.993463\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n \"ts\"=\"2026-04-10 01:37:25.995876\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n \"ts\"=\"2026-04-10 01:38:25.997941\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n \"ts\"=\"2026-04-10 01:39:26.000365\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n \"ts\"=\"2026-04-10 01:40:26.002150\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n \"ts\"=\"2026-04-10 01:41:26.004219\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n \"ts\"=\"2026-04-10 01:42:27.369138\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=118 \"elapsed\"=\"1h57m2s\"\n STEP: creating resource group @ 04/10/26 01:42:27.369\n \"ts\"=\"2026-04-10 01:42:27.369233\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-stable-4-20-c5n4s2\"\n STEP: creating customer resources @ 04/10/26 01:42:31.433\n STEP: creating the HCP cluster with version '4.20' on stable channel @ 04/10/26 01:43:32.288\n \"ts\"=\"2026-04-10 01:43:32.288564\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-stable-4-20-4vbjtk\" \"resourceGroup\"=\"rg-stable-4-20-c5n4s2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"fa273c69df13eb0ad77021ab44359762adc447d2a598d98caaf7197a9b8d7c1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","failed_at":"e2e","signature_id":"4b476ec19b339bba387ec9743367a687655cf83e55ffc3d32165311f768789cf","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000906510\u003e: \n failed to create HCP cluster ea-list7qgk62, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list7qgk62 in resource group ea-list-dsv98b, error: failed waiting for cluster=\"ea-list7qgk62\" in resourcegroup=\"ea-list-dsv98b\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list7qgk62 in resource group ea-list-dsv98b, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000906510\u003e: \n failed to create HCP cluster ea-list7qgk62, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list7qgk62 in resource group ea-list-dsv98b, error: failed waiting for cluster=\"ea-list7qgk62\" in resourcegroup=\"ea-list-dsv98b\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-list7qgk62 in resource group ea-list-dsv98b, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w2ndls\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-w2ndls, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":33,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":33,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042382069392216064","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-09T23:20:05Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e01c09b4dbffb8eef73ed18b06c631c529a4003bd458df4b5884ad3c3347389b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042382069392216064","occurred_at":"2026-04-09T23:20:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042373239870590976","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-09T22:45:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042372459289645056","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-09T22:41:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042328490195292160","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-09T19:47:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-09T10:43:15Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"04da8437c592c248d88ef130ff93c19493a5ffab68c143f909ab9dbe2e8733f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","failed_at":"e2e","signature_id":"68b4f79c4388b2a38f2fc7f55a3630604102f569c8dd5b906c95a4901a1ac61c","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:165]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a4e0c0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-vpz2m5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:165]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a4e0c0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-vpz2m5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"202990fad985b034f65de7d42b541714034af3c783d940bc92c58bf578cba650","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","failed_at":"e2e","signature_id":"aa2f8fa3e1bb2876e9c5056d271c2441da23870796e6387dd2d2709f791ed027","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008d92c0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-ddwscf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008d92c0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-ddwscf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"40e324e28a8cb8d56c2a0b9f3d5adc7e5c1c761e74c51988a99cca78ebf22ac7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"65ba4aa6d38ec2f219a95d9de0a035d130a9b0aa141551ea83bea4153228fc16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","failed_at":"e2e","signature_id":"87ddbde6dce1408bf8bb9c3fe0aef6b8db9623c90da591e311e5dcaaae1ec7c3","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:141]: Unexpected error:\n \u003c*exported.ResponseError | 0xc001404990\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-rclvn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:141]: Unexpected error:\n \u003c*exported.ResponseError | 0xc001404990\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-rclvn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"683173d3a6520478fe76027389479002ac2d2e1044c6fdc87add183d29683bd7","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention"}},{"row_id":"a89ef7577597341f8fa28306e31dae35036d4a4dec4deeec698f5729e53bf31d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","failed_at":"e2e","signature_id":"97312cc094263b3ba5463d178634f674da6726b6e062cecc0738c80b9e6d6fbf","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016011a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2668zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-wkpcsh/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016011a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-2668zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-wkpcsh/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad740faba9b7192a390c1cf0f240a46b5c5f643f128d7e9baa08c16c26076dc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","failed_at":"e2e","signature_id":"288c8d145446a41254ecea6baff1c303123323830880407686bcc2a82988756f","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001418660\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-k2bkdp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001418660\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-k2bkdp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e99d354843e84967282d12e5bfbaa1763483f7f0ffb34f7236eeb9be0793f6b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","failed_at":"e2e","signature_id":"c44640292de911bca3ae3026f422b2d682df399df9a22d6e4536f3e7e6da6324","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00141aa20\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-5jk444/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00141aa20\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-5jk444/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","683173d3a6520478fe76027389479002ac2d2e1044c6fdc87add183d29683bd7","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-09T02:13:13Z"},"failed_at":["e2e","unknown"],"failed_test_count":10,"occurrences":[{"row_id":"0efeacce33cd2c0bf8cbb847fc57bf71257d9299954ad60abf98858fa1133da5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"0a2db9b0c728d07e9e80df888d3f0ca03fe60b12602e85ef9065fba88fc78ab8","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"tity containers but only 0 are assigned\" \"attempt\"=21 \"retryIn\"=\"1m0s\" \"elapsed\"=\"20m0s\"\n \"ts\"=\"2026-04-09 03:19:42.382195\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=22 \"retryIn\"=\"1m0s\" \"elapsed\"=\"21m0s\"\n \"ts\"=\"2026-04-09 03:20:42.384493\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=23 \"retryIn\"=\"1m0s\" \"elapsed\"=\"22m0s\"\n \"ts\"=\"2026-04-09 03:21:42.387621\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=24 \"retryIn\"=\"1m0s\" \"elapsed\"=\"23m0s\"\n \"ts\"=\"2026-04-09 03:22:42.391394\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=25 \"retryIn\"=\"1m0s\" \"elapsed\"=\"24m0s\"\n \"ts\"=\"2026-04-09 03:23:42.393047\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=26 \"retryIn\"=\"1m0s\" \"elapsed\"=\"25m0s\"\n \"ts\"=\"2026-04-09 03:24:42.397053\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=27 \"retryIn\"=\"1m0s\" \"elapsed\"=\"26m0s\"\n \"ts\"=\"2026-04-09 03:25:42.399495\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=28 \"retryIn\"=\"1m0s\" \"elapsed\"=\"27m0s\"\n \"ts\"=\"2026-04-09 03:26:42.402131\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=29 \"retryIn\"=\"1m0s\" \"elapsed\"=\"28m0s\"\n \"ts\"=\"2026-04-09 03:27:42.404280\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=30 \"retryIn\"=\"1m0s\" \"elapsed\"=\"29m0s\"\n \"ts\"=\"2026-04-09 03:28:42.406960\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=31 \"retryIn\"=\"1m0s\" \"elapsed\"=\"30m0s\"\n \"ts\"=\"2026-04-09 03:29:42.408890\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=32 \"retryIn\"=\"1m0s\" \"elapsed\"=\"31m0s\"\n \"ts\"=\"2026-04-09 03:30:42.411369\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=33 \"retryIn\"=\"1m0s\" \"elapsed\"=\"32m0s\"\n \"ts\"=\"2026-04-09 03:31:42.412850\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=34 \"retryIn\"=\"1m0s\" \"elapsed\"=\"33m0s\"\n \"ts\"=\"2026-04-09 03:32:42.421462\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=35 \"elapsed\"=\"34m0s\"\n STEP: checking if the region supports availability zones @ 04/09/26 03:32:42.421\n STEP: creating a resource group @ 04/09/26 03:32:44.196\n \"ts\"=\"2026-04-09 03:32:44.196154\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-6svcw6\"\n STEP: creating cluster parameters @ 04/09/26 03:32:47.439\n STEP: creating customer resources @ 04/09/26 03:32:47.439\n STEP: creating the HCP cluster @ 04/09/26 03:33:26.766\n \"ts\"=\"2026-04-09 03:33:26.766493\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-6svcw6\"\n STEP: verifying the cluster has default autoscaling parameters @ 04/09/26 03:43:59.519\n STEP: getting admin credentials for the cluster @ 04/09/26 03:44:05.57\n STEP: creating the AZ nodepool with 500 max replicas @ 04/09/26 03:46:41.044\n STEP: verifying the AZ nodepool has the correct autoscaling configuration @ 04/09/26 04:06:10.866\n STEP: verifying node count after AZ nodepool creation @ 04/09/26 04:06:11.096\n STEP: updating the AZ nodepool max replicas from 500 to 4 before creating the next nodepool @ 04/09/26 04:06:11.701\n STEP: verifying the AZ nodepool max replicas was updated to 4 @ 04/09/26 04:10:03.502\n STEP: creating the no-AZ nodepool with 200 max replicas @ 04/09/26 04:10:03.786\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ntity containers but only 0 are assigned\" \"attempt\"=21 \"retryIn\"=\"1m0s\" \"elapsed\"=\"20m0s\"\n \"ts\"=\"2026-04-09 03:19:42.382195\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=22 \"retryIn\"=\"1m0s\" \"elapsed\"=\"21m0s\"\n \"ts\"=\"2026-04-09 03:20:42.384493\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=23 \"retryIn\"=\"1m0s\" \"elapsed\"=\"22m0s\"\n \"ts\"=\"2026-04-09 03:21:42.387621\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=24 \"retryIn\"=\"1m0s\" \"elapsed\"=\"23m0s\"\n \"ts\"=\"2026-04-09 03:22:42.391394\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=25 \"retryIn\"=\"1m0s\" \"elapsed\"=\"24m0s\"\n \"ts\"=\"2026-04-09 03:23:42.393047\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=26 \"retryIn\"=\"1m0s\" \"elapsed\"=\"25m0s\"\n \"ts\"=\"2026-04-09 03:24:42.397053\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=27 \"retryIn\"=\"1m0s\" \"elapsed\"=\"26m0s\"\n \"ts\"=\"2026-04-09 03:25:42.399495\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=28 \"retryIn\"=\"1m0s\" \"elapsed\"=\"27m0s\"\n \"ts\"=\"2026-04-09 03:26:42.402131\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=29 \"retryIn\"=\"1m0s\" \"elapsed\"=\"28m0s\"\n \"ts\"=\"2026-04-09 03:27:42.404280\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=30 \"retryIn\"=\"1m0s\" \"elapsed\"=\"29m0s\"\n \"ts\"=\"2026-04-09 03:28:42.406960\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=31 \"retryIn\"=\"1m0s\" \"elapsed\"=\"30m0s\"\n \"ts\"=\"2026-04-09 03:29:42.408890\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=32 \"retryIn\"=\"1m0s\" \"elapsed\"=\"31m0s\"\n \"ts\"=\"2026-04-09 03:30:42.411369\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=33 \"retryIn\"=\"1m0s\" \"elapsed\"=\"32m0s\"\n \"ts\"=\"2026-04-09 03:31:42.412850\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=34 \"retryIn\"=\"1m0s\" \"elapsed\"=\"33m0s\"\n \"ts\"=\"2026-04-09 03:32:42.421462\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=35 \"elapsed\"=\"34m0s\"\n STEP: checking if the region supports availability zones @ 04/09/26 03:32:42.421\n STEP: creating a resource group @ 04/09/26 03:32:44.196\n \"ts\"=\"2026-04-09 03:32:44.196154\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-6svcw6\"\n STEP: creating cluster parameters @ 04/09/26 03:32:47.439\n STEP: creating customer resources @ 04/09/26 03:32:47.439\n STEP: creating the HCP cluster @ 04/09/26 03:33:26.766\n \"ts\"=\"2026-04-09 03:33:26.766493\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-6svcw6\"\n STEP: verifying the cluster has default autoscaling parameters @ 04/09/26 03:43:59.519\n STEP: getting admin credentials for the cluster @ 04/09/26 03:44:05.57\n STEP: creating the AZ nodepool with 500 max replicas @ 04/09/26 03:46:41.044\n STEP: verifying the AZ nodepool has the correct autoscaling configuration @ 04/09/26 04:06:10.866\n STEP: verifying node count after AZ nodepool creation @ 04/09/26 04:06:11.096\n STEP: updating the AZ nodepool max replicas from 500 to 4 before creating the next nodepool @ 04/09/26 04:06:11.701\n STEP: verifying the AZ nodepool max replicas was updated to 4 @ 04/09/26 04:10:03.502\n STEP: creating the no-AZ nodepool with 200 max replicas @ 04/09/26 04:10:03.786\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"158c1c304db148748f1990351f870f240d53b9232f8db403a7a49f1f674c3eb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"a0eb876f1dcc8d31439a1e9ac1c75b37fa3dc26ff9193f4bfa7d679cba33c237","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000620120\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2r9cp8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2r9cp8, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000620120\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2r9cp8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-2r9cp8, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded"}},{"row_id":"29f379fc40ab018ffe616fb0a6647eb2b7def3730786912081a3ae23964230f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"389055155e0e9e664a2b8bbcab0e62ebe98717b50141fe5e2fc8bd747cb12977","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"etup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:169\n \"ts\"=\"2026-04-09 03:19:22.190931\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-use-workload-identity-via-the-cluster-OIDC-issuer-URL|pid:930\"\n \"ts\"=\"2026-04-09 03:19:22.197971\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/09/26 03:19:22.198\n \"ts\"=\"2026-04-09 03:19:22.198017\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\n STEP: creating cluster parameters @ 04/09/26 03:19:26.762\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/09/26 03:19:26.762\n STEP: creating the HCP cluster @ 04/09/26 03:20:04.902\n \"ts\"=\"2026-04-09 03:20:04.903044\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"oidc-wi-cluster\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\n STEP: getting the cluster's OIDC issuer URL @ 04/09/26 03:40:23.92\n Cluster OIDC issuer URL: https://uksouth.oic.aro-hcp.azure.com/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2picpq66ab9mj7d4bgfs5gp11o5je650\n STEP: getting admin credentials @ 04/09/26 03:40:26.284\n STEP: ensuring the cluster is viable @ 04/09/26 03:43:37.288\n STEP: creating the node pool @ 04/09/26 03:43:37.893\n STEP: verifying nodes are ready @ 04/09/26 03:56:28.271\n STEP: creating a user-assigned managed identity for the test @ 04/09/26 03:56:28.871\n Created managed identity with client ID: cf369737-e848-46a4-ab5c-6d8bd20486a3\n STEP: creating a federated identity credential for the service account @ 04/09/26 03:56:32.436\n Created federated identity credential with issuer: https://uksouth.oic.aro-hcp.azure.com/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2picpq66ab9mj7d4bgfs5gp11o5je650, subject: system:serviceaccount:e2e-oidc-wi:wi-test-sa\n STEP: creating the test namespace and service account in the cluster @ 04/09/26 03:56:35.234\n STEP: waiting for namespace UID range annotation to be set by the cluster-policy-controller @ 04/09/26 03:56:35.428\n STEP: creating a pod that authenticates to Azure using federated workload identity credentials @ 04/09/26 03:56:40.62\nI0409 03:56:40.836741 930 warnings.go:110] \"Warning: would violate PodSecurity \\\"restricted:latest\\\": runAsNonRoot != true (pod or container \\\"azure-cli\\\" must set securityContext.runAsNonRoot=true), runAsUser=0 (container \\\"azure-cli\\\" must not set runAsUser=0)\"\n STEP: waiting for the pod to complete successfully @ 04/09/26 03:56:40.836\n Pod state transitioned from 'Pending' to 'Running'\n Pod state transitioned from 'Running' to 'Succeeded'\n Workload identity authentication succeeded. Pod completed successfully.\n \"ts\"=\"2026-04-09 03:57:31.424893\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 03:57:31.425059\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-oidc-wi-cluster-wfpm58\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\n \"ts\"=\"2026-04-09 03:57:31.783064\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-oidc-wi-cluster-wfpm58\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 03:57:33.276552\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-f2883517cea1430b4915f04acc052b067a0c30de1b37a57f443ec7d05cd4dc52.yaml.gz\"\n \"ts\"=\"2026-04-09 03:57:33.276672\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-f2883517cea1430b4915f04acc052b067a0c30de1b37a57f443ec7d05cd4dc52.yaml.gz\"\n \"ts\"=\"2026-04-09 03:57:33.276738\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 03:57:33.278253\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-71\"\n \"ts\"=\"2026-04-09 03:57:33.278266\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\n \"ts\"=\"2026-04-09 03:57:33.416422\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-71\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 03:57:33.540480\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-71\" \"deployment\"=\"msi-deployment-71\"\n \"ts\"=\"2026-04-09 03:57:33.829957\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"oidc-wi-pvl2d2\" \"deployment\"=\"pooledNonMsiScopedAssignments-s6kwp7xg43dbi\"\n \"ts\"=\"2026-04-09 03:57:33.973325\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"oidc-wi-pvl2d2\" \"deployment\"=\"customer-infra-oidc-wi-cluster-wfpm58\"\n \"ts\"=\"2026-04-09 03:57:34.121191\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 03:57:34.121312\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 03:57:34.121454\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\netup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:169\n \"ts\"=\"2026-04-09 03:19:22.190931\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-use-workload-identity-via-the-cluster-OIDC-issuer-URL|pid:930\"\n \"ts\"=\"2026-04-09 03:19:22.197971\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/09/26 03:19:22.198\n \"ts\"=\"2026-04-09 03:19:22.198017\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\n STEP: creating cluster parameters @ 04/09/26 03:19:26.762\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/09/26 03:19:26.762\n STEP: creating the HCP cluster @ 04/09/26 03:20:04.902\n \"ts\"=\"2026-04-09 03:20:04.903044\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"oidc-wi-cluster\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\n STEP: getting the cluster's OIDC issuer URL @ 04/09/26 03:40:23.92\n Cluster OIDC issuer URL: https://uksouth.oic.aro-hcp.azure.com/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2picpq66ab9mj7d4bgfs5gp11o5je650\n STEP: getting admin credentials @ 04/09/26 03:40:26.284\n STEP: ensuring the cluster is viable @ 04/09/26 03:43:37.288\n STEP: creating the node pool @ 04/09/26 03:43:37.893\n STEP: verifying nodes are ready @ 04/09/26 03:56:28.271\n STEP: creating a user-assigned managed identity for the test @ 04/09/26 03:56:28.871\n Created managed identity with client ID: cf369737-e848-46a4-ab5c-6d8bd20486a3\n STEP: creating a federated identity credential for the service account @ 04/09/26 03:56:32.436\n Created federated identity credential with issuer: https://uksouth.oic.aro-hcp.azure.com/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2picpq66ab9mj7d4bgfs5gp11o5je650, subject: system:serviceaccount:e2e-oidc-wi:wi-test-sa\n STEP: creating the test namespace and service account in the cluster @ 04/09/26 03:56:35.234\n STEP: waiting for namespace UID range annotation to be set by the cluster-policy-controller @ 04/09/26 03:56:35.428\n STEP: creating a pod that authenticates to Azure using federated workload identity credentials @ 04/09/26 03:56:40.62\nI0409 03:56:40.836741 930 warnings.go:110] \"Warning: would violate PodSecurity \\\"restricted:latest\\\": runAsNonRoot != true (pod or container \\\"azure-cli\\\" must set securityContext.runAsNonRoot=true), runAsUser=0 (container \\\"azure-cli\\\" must not set runAsUser=0)\"\n STEP: waiting for the pod to complete successfully @ 04/09/26 03:56:40.836\n Pod state transitioned from 'Pending' to 'Running'\n Pod state transitioned from 'Running' to 'Succeeded'\n Workload identity authentication succeeded. Pod completed successfully.\n \"ts\"=\"2026-04-09 03:57:31.424893\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 03:57:31.425059\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-oidc-wi-cluster-wfpm58\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\n \"ts\"=\"2026-04-09 03:57:31.783064\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-oidc-wi-cluster-wfpm58\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 03:57:33.276552\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-f2883517cea1430b4915f04acc052b067a0c30de1b37a57f443ec7d05cd4dc52.yaml.gz\"\n \"ts\"=\"2026-04-09 03:57:33.276672\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-f2883517cea1430b4915f04acc052b067a0c30de1b37a57f443ec7d05cd4dc52.yaml.gz\"\n \"ts\"=\"2026-04-09 03:57:33.276738\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 03:57:33.278253\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-71\"\n \"ts\"=\"2026-04-09 03:57:33.278266\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\n \"ts\"=\"2026-04-09 03:57:33.416422\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-71\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 03:57:33.540480\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-71\" \"deployment\"=\"msi-deployment-71\"\n \"ts\"=\"2026-04-09 03:57:33.829957\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"oidc-wi-pvl2d2\" \"deployment\"=\"pooledNonMsiScopedAssignments-s6kwp7xg43dbi\"\n \"ts\"=\"2026-04-09 03:57:33.973325\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"oidc-wi-pvl2d2\" \"deployment\"=\"customer-infra-oidc-wi-cluster-wfpm58\"\n \"ts\"=\"2026-04-09 03:57:34.121191\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 03:57:34.121312\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 03:57:34.121454\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"oidc-wi-pvl2d2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"5845bba5eae759f1892538597fbfb985aa28802d9037970b10dd74673c2f79b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"35df135bd95fbdb3da8af53544341ab557e41e5de58dcd707fcc3fd1aa7d8ba8","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775704466 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\nW0409 03:14:26.771406 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0409 03:14:26.888642 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-09 03:14:26.889032\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:893\"\n \"ts\"=\"2026-04-09 03:14:26.895878\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/09/26 03:14:26.895\n \"ts\"=\"2026-04-09 03:14:26.895962\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-rtdzxc-rgpqsm\"\n STEP: creating cluster parameters at control plane version @ 04/09/26 03:14:30.733\nW0409 03:14:30.812060 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/09/26 03:14:30.812\n STEP: creating the HCP cluster with version 4.21.7 @ 04/09/26 03:15:20.256\n \"ts\"=\"2026-04-09 03:15:20.256746\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-rtdzxc\" \"resourceGroup\"=\"rg-np-version-upgrade-rtdzxc-rgpqsm\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/09/26 03:39:18.605\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/09/26 03:53:11.727\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/09/26 04:01:25.78\nW0409 04:01:25.907854 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: capturing node release images before upgrade @ 04/09/26 04:01:25.907\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/09/26 04:01:26.096\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/09/26 04:02:42.093\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775704466 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\nW0409 03:14:26.771406 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0409 03:14:26.888642 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-09 03:14:26.889032\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:893\"\n \"ts\"=\"2026-04-09 03:14:26.895878\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/09/26 03:14:26.895\n \"ts\"=\"2026-04-09 03:14:26.895962\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-rtdzxc-rgpqsm\"\n STEP: creating cluster parameters at control plane version @ 04/09/26 03:14:30.733\nW0409 03:14:30.812060 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/09/26 03:14:30.812\n STEP: creating the HCP cluster with version 4.21.7 @ 04/09/26 03:15:20.256\n \"ts\"=\"2026-04-09 03:15:20.256746\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-rtdzxc\" \"resourceGroup\"=\"rg-np-version-upgrade-rtdzxc-rgpqsm\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/09/26 03:39:18.605\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/09/26 03:53:11.727\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/09/26 04:01:25.78\nW0409 04:01:25.907854 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: capturing node release images before upgrade @ 04/09/26 04:01:25.907\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/09/26 04:01:26.096\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/09/26 04:02:42.093\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"5e8507aebf8c981fdc3d676a39b1519147c25cec6608fb9b7967852fe53e574d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-09T04:15:36Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"75a3c4f4dfed9c474bbaa318c5c1bb10299a046d286dfd4872de2019e69a67ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"8214c0ce13cb6991671914fc7157483e675c7611e2c51702aee60546ac6d5553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"905b4519fd2ae631d09bec2ad336fe05d492e64ad60a1c4840fb63249115793c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"-fsmm4\" \"labels\"={\"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"failure-domain.beta.kubernetes.io/region\"=\"uksouth\" \"hypershift.openshift.io/nodePool\"=\"k5s4w5q8w1f4e5m-np-1\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-fsmm4\" \"node-role.kubernetes.io/worker\"=\"\" \"kubernetes.io/os\"=\"linux\" \"topology.kubernetes.io/region\"=\"uksouth\" \"beta.kubernetes.io/arch\"=\"amd64\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n \"ts\"=\"2026-04-09 04:13:15.242897\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-k6cjg\" \"labels\"={\"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"uksouth\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/os\"=\"linux\" \"kubernetes.io/hostname\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-k6cjg\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/nodePool\"=\"k5s4w5q8w1f4e5m-np-1\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"key1\"=\"value1\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"uksouth\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-09 04:13:15.242918\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-lwmnr\" \"labels\"={\"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"uksouth\" \"hypershift.openshift.io/nodePool\"=\"k5s4w5q8w1f4e5m-np-1\" \"key2\"=\"value2\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.kubernetes.io/region\"=\"uksouth\" \"topology.kubernetes.io/zone\"=\"1\" \"node.openshift.io/os_id\"=\"rhel\" \"beta.kubernetes.io/arch\"=\"amd64\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-lwmnr\" \"topology.disk.csi.azure.com/zone\"=\"\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n STEP: verifying original labels persist on 3 nodes @ 04/09/26 04:13:15.242\n STEP: verifying original taints persist on 2 nodes @ 04/09/26 04:13:15.243\n \"ts\"=\"2026-04-09 04:13:15.243171\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 04:13:15.243246\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-2hzkp9\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\"\n \"ts\"=\"2026-04-09 04:13:15.414442\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-2hzkp9\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 04:13:16.509458\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-09 04:13:16.509599\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-09 04:13:16.509657\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 04:13:16.511230\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\"\n \"ts\"=\"2026-04-09 04:13:16.511263\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\"\n \"ts\"=\"2026-04-09 04:13:16.668175\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-09 04:13:16.673052\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 04:13:16.799826\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-2hzkp9\"\n \"ts\"=\"2026-04-09 04:13:16.834046\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"msi-deployment-4\"\n \"ts\"=\"2026-04-09 04:13:16.972023\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 04:13:16.972168\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 04:13:16.972298\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n-fsmm4\" \"labels\"={\"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"failure-domain.beta.kubernetes.io/region\"=\"uksouth\" \"hypershift.openshift.io/nodePool\"=\"k5s4w5q8w1f4e5m-np-1\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-fsmm4\" \"node-role.kubernetes.io/worker\"=\"\" \"kubernetes.io/os\"=\"linux\" \"topology.kubernetes.io/region\"=\"uksouth\" \"beta.kubernetes.io/arch\"=\"amd64\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n \"ts\"=\"2026-04-09 04:13:15.242897\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-k6cjg\" \"labels\"={\"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"uksouth\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/os\"=\"linux\" \"kubernetes.io/hostname\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-k6cjg\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/nodePool\"=\"k5s4w5q8w1f4e5m-np-1\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"key1\"=\"value1\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"uksouth\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-09 04:13:15.242918\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-lwmnr\" \"labels\"={\"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"uksouth\" \"hypershift.openshift.io/nodePool\"=\"k5s4w5q8w1f4e5m-np-1\" \"key2\"=\"value2\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.kubernetes.io/region\"=\"uksouth\" \"topology.kubernetes.io/zone\"=\"1\" \"node.openshift.io/os_id\"=\"rhel\" \"beta.kubernetes.io/arch\"=\"amd64\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"k5s4w5q8w1f4e5m-np-1-vs69q-lwmnr\" \"topology.disk.csi.azure.com/zone\"=\"\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n STEP: verifying original labels persist on 3 nodes @ 04/09/26 04:13:15.242\n STEP: verifying original taints persist on 2 nodes @ 04/09/26 04:13:15.243\n \"ts\"=\"2026-04-09 04:13:15.243171\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 04:13:15.243246\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-2hzkp9\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\"\n \"ts\"=\"2026-04-09 04:13:15.414442\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-2hzkp9\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 04:13:16.509458\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-09 04:13:16.509599\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-09 04:13:16.509657\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 04:13:16.511230\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\"\n \"ts\"=\"2026-04-09 04:13:16.511263\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\"\n \"ts\"=\"2026-04-09 04:13:16.668175\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-09 04:13:16.673052\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 04:13:16.799826\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-2hzkp9\"\n \"ts\"=\"2026-04-09 04:13:16.834046\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"msi-deployment-4\"\n \"ts\"=\"2026-04-09 04:13:16.972023\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 04:13:16.972168\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 04:13:16.972298\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-fs4hs8\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"dd70334bb4af5267ff368baa3464d7a88697326a6353a95921c43d6d1956bdcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"2769ee201200e30cdfaa3da531246e1754a9bebacbf8e5667de0496fbc64cba4","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:168]: Unexpected error:\n \u003c*url.Error | 0xc001248ab0\u003e: \n Get \"https://api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.q6i1z1a9v3j3w8r.qgw8.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:168]: Unexpected error:\n \u003c*url.Error | 0xc001248ab0\u003e: \n Get \"https://api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.q6i1z1a9v3j3w8r.qgw8.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io"}},{"row_id":"e12d0de16a44a239cf76c72ddd2bf7fe8a0936de155b5c750595fa0e0f768621","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"691bb7eb88df394a0a911b08820aeb36de06814b9281babcaddd7c570f135e34","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n \u003c*errors.joinError | 0xc0013be210\u003e: \n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyBasicAccess failed: failed to list services: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n \u003c*errors.joinError | 0xc0013be210\u003e: \n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyBasicAccess failed: failed to list services: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io"}},{"row_id":"f5df2f96a545079eeec6a75eaffd29823c90164de5c3a582a125a15fd4a28abc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-09T02:00:33Z"},"failed_at":["e2e","unknown"],"failed_test_count":11,"occurrences":[{"row_id":"01fe8fb32517f11f0f873e54f76754fb2c59b4cd4d8ecb70be9e1f728b9294a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"809d7844f1a48c70c3423a63cecd4101fb58ab0d6a342e98acba6ff46c3c4077","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"1 identity containers but only 0 are assigned\" \"attempt\"=9 \"retryIn\"=\"1m0s\" \"elapsed\"=\"8m0s\"\n \"ts\"=\"2026-04-09 03:06:10.991148\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=10 \"retryIn\"=\"1m0s\" \"elapsed\"=\"9m0s\"\n \"ts\"=\"2026-04-09 03:07:10.993390\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=11 \"retryIn\"=\"1m0s\" \"elapsed\"=\"10m0s\"\n \"ts\"=\"2026-04-09 03:08:10.995597\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=12 \"retryIn\"=\"1m0s\" \"elapsed\"=\"11m0s\"\n \"ts\"=\"2026-04-09 03:09:10.998072\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=13 \"retryIn\"=\"1m0s\" \"elapsed\"=\"12m0s\"\n \"ts\"=\"2026-04-09 03:10:10.999567\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=14 \"retryIn\"=\"1m0s\" \"elapsed\"=\"13m0s\"\n \"ts\"=\"2026-04-09 03:11:11.001831\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=15 \"retryIn\"=\"1m0s\" \"elapsed\"=\"14m0s\"\n \"ts\"=\"2026-04-09 03:12:11.003605\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=16 \"retryIn\"=\"1m0s\" \"elapsed\"=\"15m0s\"\n \"ts\"=\"2026-04-09 03:13:11.005877\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=17 \"retryIn\"=\"1m0s\" \"elapsed\"=\"16m0s\"\n \"ts\"=\"2026-04-09 03:14:11.007936\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=18 \"retryIn\"=\"1m0s\" \"elapsed\"=\"17m0s\"\n \"ts\"=\"2026-04-09 03:15:11.697320\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=19 \"retryIn\"=\"1m0s\" \"elapsed\"=\"18m1s\"\n \"ts\"=\"2026-04-09 03:16:11.699773\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=20 \"retryIn\"=\"1m0s\" \"elapsed\"=\"19m1s\"\n \"ts\"=\"2026-04-09 03:17:11.702809\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=21 \"retryIn\"=\"1m0s\" \"elapsed\"=\"20m1s\"\n \"ts\"=\"2026-04-09 03:18:11.705098\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=22 \"retryIn\"=\"1m0s\" \"elapsed\"=\"21m1s\"\n \"ts\"=\"2026-04-09 03:19:11.715330\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=23 \"elapsed\"=\"22m1s\"\n STEP: checking if the region supports availability zones @ 04/09/26 03:19:11.715\n STEP: creating a resource group @ 04/09/26 03:19:13.671\n \"ts\"=\"2026-04-09 03:19:13.671663\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-dkwlpg\"\n STEP: creating cluster parameters @ 04/09/26 03:19:17.02\n STEP: creating customer resources @ 04/09/26 03:19:17.02\n STEP: creating the HCP cluster @ 04/09/26 03:20:05.95\n \"ts\"=\"2026-04-09 03:20:05.950751\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-dkwlpg\"\n STEP: verifying the cluster has default autoscaling parameters @ 04/09/26 03:37:52.738\n STEP: getting admin credentials for the cluster @ 04/09/26 03:37:53.37\n STEP: creating the AZ nodepool with 500 max replicas @ 04/09/26 03:44:04.67\n STEP: verifying the AZ nodepool has the correct autoscaling configuration @ 04/09/26 03:53:13.983\n STEP: verifying node count after AZ nodepool creation @ 04/09/26 03:53:14.197\n STEP: updating the AZ nodepool max replicas from 500 to 4 before creating the next nodepool @ 04/09/26 03:53:14.727\n STEP: verifying the AZ nodepool max replicas was updated to 4 @ 04/09/26 03:59:30.841\n STEP: creating the no-AZ nodepool with 200 max replicas @ 04/09/26 03:59:31.163\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n1 identity containers but only 0 are assigned\" \"attempt\"=9 \"retryIn\"=\"1m0s\" \"elapsed\"=\"8m0s\"\n \"ts\"=\"2026-04-09 03:06:10.991148\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=10 \"retryIn\"=\"1m0s\" \"elapsed\"=\"9m0s\"\n \"ts\"=\"2026-04-09 03:07:10.993390\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=11 \"retryIn\"=\"1m0s\" \"elapsed\"=\"10m0s\"\n \"ts\"=\"2026-04-09 03:08:10.995597\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=12 \"retryIn\"=\"1m0s\" \"elapsed\"=\"11m0s\"\n \"ts\"=\"2026-04-09 03:09:10.998072\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=13 \"retryIn\"=\"1m0s\" \"elapsed\"=\"12m0s\"\n \"ts\"=\"2026-04-09 03:10:10.999567\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=14 \"retryIn\"=\"1m0s\" \"elapsed\"=\"13m0s\"\n \"ts\"=\"2026-04-09 03:11:11.001831\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=15 \"retryIn\"=\"1m0s\" \"elapsed\"=\"14m0s\"\n \"ts\"=\"2026-04-09 03:12:11.003605\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=16 \"retryIn\"=\"1m0s\" \"elapsed\"=\"15m0s\"\n \"ts\"=\"2026-04-09 03:13:11.005877\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=17 \"retryIn\"=\"1m0s\" \"elapsed\"=\"16m0s\"\n \"ts\"=\"2026-04-09 03:14:11.007936\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=18 \"retryIn\"=\"1m0s\" \"elapsed\"=\"17m0s\"\n \"ts\"=\"2026-04-09 03:15:11.697320\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=19 \"retryIn\"=\"1m0s\" \"elapsed\"=\"18m1s\"\n \"ts\"=\"2026-04-09 03:16:11.699773\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=20 \"retryIn\"=\"1m0s\" \"elapsed\"=\"19m1s\"\n \"ts\"=\"2026-04-09 03:17:11.702809\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=21 \"retryIn\"=\"1m0s\" \"elapsed\"=\"20m1s\"\n \"ts\"=\"2026-04-09 03:18:11.705098\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=22 \"retryIn\"=\"1m0s\" \"elapsed\"=\"21m1s\"\n \"ts\"=\"2026-04-09 03:19:11.715330\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=23 \"elapsed\"=\"22m1s\"\n STEP: checking if the region supports availability zones @ 04/09/26 03:19:11.715\n STEP: creating a resource group @ 04/09/26 03:19:13.671\n \"ts\"=\"2026-04-09 03:19:13.671663\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-dkwlpg\"\n STEP: creating cluster parameters @ 04/09/26 03:19:17.02\n STEP: creating customer resources @ 04/09/26 03:19:17.02\n STEP: creating the HCP cluster @ 04/09/26 03:20:05.95\n \"ts\"=\"2026-04-09 03:20:05.950751\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-dkwlpg\"\n STEP: verifying the cluster has default autoscaling parameters @ 04/09/26 03:37:52.738\n STEP: getting admin credentials for the cluster @ 04/09/26 03:37:53.37\n STEP: creating the AZ nodepool with 500 max replicas @ 04/09/26 03:44:04.67\n STEP: verifying the AZ nodepool has the correct autoscaling configuration @ 04/09/26 03:53:13.983\n STEP: verifying node count after AZ nodepool creation @ 04/09/26 03:53:14.197\n STEP: updating the AZ nodepool max replicas from 500 to 4 before creating the next nodepool @ 04/09/26 03:53:14.727\n STEP: verifying the AZ nodepool max replicas was updated to 4 @ 04/09/26 03:59:30.841\n STEP: creating the no-AZ nodepool with 200 max replicas @ 04/09/26 03:59:31.163\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"2e453af1d87887c855eec6d2f005efa2ca8a870f76e1ed7ed45e10e5cea521d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"0e3db37633accfbbb3e81a3af573a728d2faa0040f7694a8ebd9aa388f5fbf5f","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"=================\n Random Seed: 1775703951 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\nW0409 03:05:51.455222 904 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0409 03:05:51.559512 904 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-09 03:05:51.559836\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:904\"\n \"ts\"=\"2026-04-09 03:05:51.567639\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/09/26 03:05:51.567\n \"ts\"=\"2026-04-09 03:05:51.567727\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\n STEP: creating cluster parameters at control plane version @ 04/09/26 03:05:55.822\n STEP: creating customer resources @ 04/09/26 03:05:55.926\n STEP: creating the HCP cluster with version 4.20.17 @ 04/09/26 03:06:44.311\n \"ts\"=\"2026-04-09 03:06:44.311574\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-ql4mqn\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/09/26 03:17:47.84\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/09/26 03:29:40.742\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/09/26 03:39:15.385\n STEP: capturing node release images before upgrade @ 04/09/26 03:39:15.564\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/09/26 03:39:15.754\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/09/26 03:40:29.087\n STEP: verifying node pool GET still reflects the new version @ 04/09/26 04:02:35.07\n \"ts\"=\"2026-04-09 04:02:38.256269\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 04:02:38.256359\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-ql4mqn-l6q8c7\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\n \"ts\"=\"2026-04-09 04:02:38.425554\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-ql4mqn-l6q8c7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 04:02:40.358398\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-09 04:02:40.358532\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-09 04:02:40.358597\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 04:02:40.360177\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-46\"\n \"ts\"=\"2026-04-09 04:02:40.360200\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\n \"ts\"=\"2026-04-09 04:02:40.508440\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-46\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 04:02:40.542981\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\" \"deployment\"=\"pooledNonMsiScopedAssignments-cv7d7g7burna6\"\n \"ts\"=\"2026-04-09 04:02:40.631200\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-46\" \"deployment\"=\"msi-deployment-46\"\n \"ts\"=\"2026-04-09 04:02:40.690043\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-ql4mqn-l6q8c7\"\n \"ts\"=\"2026-04-09 04:02:40.862864\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 04:02:40.863018\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 04:02:40.863144\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n=================\n Random Seed: 1775703951 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\nW0409 03:05:51.455222 904 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0409 03:05:51.559512 904 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-09 03:05:51.559836\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:904\"\n \"ts\"=\"2026-04-09 03:05:51.567639\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/09/26 03:05:51.567\n \"ts\"=\"2026-04-09 03:05:51.567727\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\n STEP: creating cluster parameters at control plane version @ 04/09/26 03:05:55.822\n STEP: creating customer resources @ 04/09/26 03:05:55.926\n STEP: creating the HCP cluster with version 4.20.17 @ 04/09/26 03:06:44.311\n \"ts\"=\"2026-04-09 03:06:44.311574\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-ql4mqn\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/09/26 03:17:47.84\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/09/26 03:29:40.742\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/09/26 03:39:15.385\n STEP: capturing node release images before upgrade @ 04/09/26 03:39:15.564\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/09/26 03:39:15.754\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/09/26 03:40:29.087\n STEP: verifying node pool GET still reflects the new version @ 04/09/26 04:02:35.07\n \"ts\"=\"2026-04-09 04:02:38.256269\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 04:02:38.256359\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-ql4mqn-l6q8c7\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\n \"ts\"=\"2026-04-09 04:02:38.425554\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-ql4mqn-l6q8c7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 04:02:40.358398\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-09 04:02:40.358532\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-09 04:02:40.358597\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 04:02:40.360177\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-46\"\n \"ts\"=\"2026-04-09 04:02:40.360200\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\n \"ts\"=\"2026-04-09 04:02:40.508440\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-46\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 04:02:40.542981\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\" \"deployment\"=\"pooledNonMsiScopedAssignments-cv7d7g7burna6\"\n \"ts\"=\"2026-04-09 04:02:40.631200\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-46\" \"deployment\"=\"msi-deployment-46\"\n \"ts\"=\"2026-04-09 04:02:40.690043\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-ql4mqn-l6q8c7\"\n \"ts\"=\"2026-04-09 04:02:40.862864\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 04:02:40.863018\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 04:02:40.863144\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-ql4mqn-hbrtb8\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"2ed171c284e0cae20d664181bcf35e6a37abf461383ed3d9bce837c836fdac98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"7843d873e84cfde4df6aa09b0a6bbf5526a5df8bc96e125218f65bb90b5c80b0","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001578ae0\u003e: \n failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7nb8nk\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7nb8nk, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001578ae0\u003e: \n failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7nb8nk\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7nb8nk, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded"}},{"row_id":"7b158505ff8d7c2dcd628acca4e36bd8bdfd78876a07fc8a52586caff07619c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-09T04:05:03Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"8479f8f9a2960918bd10c659b837c6e1b5812ca6eaab0a0dcb5628b6b8975924","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"894252d83a7cec0669fe253dbbb4659f9f63936c25c5f0d636e352896ea4ce5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"b5735b60f9dfe41ca07bdf25a9ce5b92363a013abfc259e91caa0c98738482cc","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"ning results: skipping unrecognized cluster condition type \"PromQL\"\nW0409 03:00:39.275674 879 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-09 03:00:39.275951\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:879\"\n \"ts\"=\"2026-04-09 03:00:39.281953\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/09/26 03:00:39.282\n \"ts\"=\"2026-04-09 03:00:39.282084\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\n STEP: creating cluster parameters at control plane version @ 04/09/26 03:00:43.984\nW0409 03:00:44.067664 879 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/09/26 03:00:44.067\n STEP: creating the HCP cluster with version 4.21.7 @ 04/09/26 03:01:43.145\n \"ts\"=\"2026-04-09 03:01:43.145732\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gxv6r5\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/09/26 03:14:34.834\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/09/26 03:29:40.461\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/09/26 03:39:19.406\nW0409 03:39:19.533244 879 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: capturing node release images before upgrade @ 04/09/26 03:39:19.533\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/09/26 03:39:19.728\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/09/26 03:40:22.889\n STEP: verifying node pool GET still reflects the new version @ 04/09/26 04:04:29.49\n \"ts\"=\"2026-04-09 04:04:30.537024\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 04:04:30.537109\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gxv6r5-2shb8x\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\n \"ts\"=\"2026-04-09 04:04:30.880368\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-gxv6r5-2shb8x\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 04:04:32.708555\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-09 04:04:32.708690\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-09 04:04:32.708757\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 04:04:32.710584\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-21\"\n \"ts\"=\"2026-04-09 04:04:32.710604\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\n \"ts\"=\"2026-04-09 04:04:32.895588\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\" \"deployment\"=\"pooledNonMsiScopedAssignments-bpnyq4yptvztk\"\n \"ts\"=\"2026-04-09 04:04:33.040592\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-21\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 04:04:33.201672\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gxv6r5-2shb8x\"\n \"ts\"=\"2026-04-09 04:04:33.213643\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-21\" \"deployment\"=\"msi-deployment-21\"\n \"ts\"=\"2026-04-09 04:04:33.375025\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 04:04:33.375136\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 04:04:33.375250\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nning results: skipping unrecognized cluster condition type \"PromQL\"\nW0409 03:00:39.275674 879 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-09 03:00:39.275951\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:879\"\n \"ts\"=\"2026-04-09 03:00:39.281953\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/09/26 03:00:39.282\n \"ts\"=\"2026-04-09 03:00:39.282084\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\n STEP: creating cluster parameters at control plane version @ 04/09/26 03:00:43.984\nW0409 03:00:44.067664 879 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/09/26 03:00:44.067\n STEP: creating the HCP cluster with version 4.21.7 @ 04/09/26 03:01:43.145\n \"ts\"=\"2026-04-09 03:01:43.145732\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gxv6r5\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/09/26 03:14:34.834\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/09/26 03:29:40.461\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/09/26 03:39:19.406\nW0409 03:39:19.533244 879 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: capturing node release images before upgrade @ 04/09/26 03:39:19.533\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/09/26 03:39:19.728\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/09/26 03:40:22.889\n STEP: verifying node pool GET still reflects the new version @ 04/09/26 04:04:29.49\n \"ts\"=\"2026-04-09 04:04:30.537024\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 04:04:30.537109\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gxv6r5-2shb8x\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\n \"ts\"=\"2026-04-09 04:04:30.880368\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-gxv6r5-2shb8x\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 04:04:32.708555\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-09 04:04:32.708690\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-09 04:04:32.708757\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 04:04:32.710584\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-21\"\n \"ts\"=\"2026-04-09 04:04:32.710604\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\n \"ts\"=\"2026-04-09 04:04:32.895588\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\" \"deployment\"=\"pooledNonMsiScopedAssignments-bpnyq4yptvztk\"\n \"ts\"=\"2026-04-09 04:04:33.040592\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-21\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 04:04:33.201672\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gxv6r5-2shb8x\"\n \"ts\"=\"2026-04-09 04:04:33.213643\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-21\" \"deployment\"=\"msi-deployment-21\"\n \"ts\"=\"2026-04-09 04:04:33.375025\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 04:04:33.375136\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 04:04:33.375250\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gxv6r5-lslgt8\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"9f028d94e754ebf9bdd5028661fb38fe6467c02ed7cf8e21f816ded517dd84b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"039071a1f2cdd978847b2c39b4ab9740ad502f6c020afd94bf765b938f2cfe82","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:180]: Timed out after 600.000s.\nrouter-default deployment logs should be fetchable\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d582a0\u003e: \n get deployment router-default -n openshift-ingress: Get \"https://api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io:443/apis/apps/v1/namespaces/openshift-ingress/deployments/router-default\": tls: failed to verify certificate: x509: certificate is valid for api.o6n1s5y5u2a8q7c.wjok.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:180]: Timed out after 600.000s.\nrouter-default deployment logs should be fetchable\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d582a0\u003e: \n get deployment router-default -n openshift-ingress: Get \"https://api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io:443/apis/apps/v1/namespaces/openshift-ingress/deployments/router-default\": tls: failed to verify certificate: x509: certificate is valid for api.o6n1s5y5u2a8q7c.wjok.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"635a2afde9e3afd24ee6abfeb373ae222c4b51e9e9a5fed30800c3f45ff07db5","failure_pattern":"get deployment router-default -n openshift-ingress: Get \"\u003curl\u003e","search_query":"get deployment router-default -n openshift-ingress: Get \"https://api"}},{"row_id":"ad41c935b44a8f6a95ea586bc4c230940bbe2c2a5b413aaa958e367fdd2fd348","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"f2738bf06536a42aa77a9502fb7deb243a43c8d2c689ef96192d23fe73aa8ea4","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"dRequestContent\\\",\\n \\\"message\\\": \\\"Invalid value: 201: must be less than or equal to 200\\\",\\n \\\"target\\\": \\\"properties.replicas\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n STEP: creating a nodepool @ 04/09/26 03:44:13.396\n STEP: attempting to update nodepool version to higher than cluster version @ 04/09/26 03:56:28.371\n \"ts\"=\"2026-04-09 03:56:29.023343\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-wwtbdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n \\\"target\\\": \\\"properties.version.id\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n STEP: attempting to update immutable platform profile fields @ 04/09/26 03:56:29.023\n \"ts\"=\"2026-04-09 03:56:29.647891\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-wwtbdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"4 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n \"ts\"=\"2026-04-09 03:56:30.265128\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 03:56:30.265213\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-75dgnz\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\"\n \"ts\"=\"2026-04-09 03:56:30.421886\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-75dgnz\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 03:56:31.420106\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-09 03:56:31.420230\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-09 03:56:31.420289\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 03:56:31.421880\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\"\n \"ts\"=\"2026-04-09 03:56:31.421913\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\"\n \"ts\"=\"2026-04-09 03:56:31.572184\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 03:56:31.578091\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n \"ts\"=\"2026-04-09 03:56:31.700962\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\" \"deployment\"=\"msi-deployment-53\"\n \"ts\"=\"2026-04-09 03:56:31.724961\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\" \"deployment\"=\"customer-infra-negative-tests-cluster-75dgnz\"\n \"ts\"=\"2026-04-09 03:56:31.872303\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 03:56:31.872425\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 03:56:31.872555\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ndRequestContent\\\",\\n \\\"message\\\": \\\"Invalid value: 201: must be less than or equal to 200\\\",\\n \\\"target\\\": \\\"properties.replicas\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n STEP: creating a nodepool @ 04/09/26 03:44:13.396\n STEP: attempting to update nodepool version to higher than cluster version @ 04/09/26 03:56:28.371\n \"ts\"=\"2026-04-09 03:56:29.023343\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-wwtbdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n \\\"target\\\": \\\"properties.version.id\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n STEP: attempting to update immutable platform profile fields @ 04/09/26 03:56:29.023\n \"ts\"=\"2026-04-09 03:56:29.647891\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-wwtbdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"4 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n \"ts\"=\"2026-04-09 03:56:30.265128\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 03:56:30.265213\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-75dgnz\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\"\n \"ts\"=\"2026-04-09 03:56:30.421886\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-75dgnz\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 03:56:31.420106\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-09 03:56:31.420230\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-09 03:56:31.420289\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 03:56:31.421880\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\"\n \"ts\"=\"2026-04-09 03:56:31.421913\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\"\n \"ts\"=\"2026-04-09 03:56:31.572184\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 03:56:31.578091\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n \"ts\"=\"2026-04-09 03:56:31.700962\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\" \"deployment\"=\"msi-deployment-53\"\n \"ts\"=\"2026-04-09 03:56:31.724961\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\" \"deployment\"=\"customer-infra-negative-tests-cluster-75dgnz\"\n \"ts\"=\"2026-04-09 03:56:31.872303\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 03:56:31.872425\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 03:56:31.872555\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-wwtbdt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"defb8528271c9410af56eeacaa485bdb1a0efcf081cb8d4adf8921248ae6c524","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"95ffaf23bec65ec54f5a233ebbd9ba6ee14445df48b2a02c2a9c8c45aa82d42e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00157a420\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7g7qrf\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-7g7qrf, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00157a420\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7g7qrf\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-7g7qrf, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded"}},{"row_id":"e58f78a4c2b8ea408a677864eb3ee52e60ff3fa591e4f4be15917ffa393e25de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"0fb0bab9c886e219989c17756195bd6d490d3111d3d7e44bba6ea3fa77bb66c5","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"s but only 0 are assigned\" \"attempt\"=1 \"retryIn\"=\"1m0s\" \"elapsed\"=\"0s\"\n \"ts\"=\"2026-04-09 03:16:11.746501\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=2 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1m0s\"\n \"ts\"=\"2026-04-09 03:17:11.749675\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=3 \"retryIn\"=\"1m0s\" \"elapsed\"=\"2m0s\"\n \"ts\"=\"2026-04-09 03:18:11.752116\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=4 \"retryIn\"=\"1m0s\" \"elapsed\"=\"3m0s\"\n \"ts\"=\"2026-04-09 03:19:11.754307\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=5 \"retryIn\"=\"1m0s\" \"elapsed\"=\"4m0s\"\n \"ts\"=\"2026-04-09 03:20:11.756347\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=6 \"retryIn\"=\"1m0s\" \"elapsed\"=\"5m0s\"\n \"ts\"=\"2026-04-09 03:21:11.766667\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=7 \"elapsed\"=\"6m0s\"\n STEP: creating a resource group @ 04/09/26 03:21:11.766\n \"ts\"=\"2026-04-09 03:21:11.766746\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\n STEP: creating cluster parameters @ 04/09/26 03:21:16.342\n STEP: creating customer resources @ 04/09/26 03:21:16.342\n STEP: creating the HCP cluster @ 04/09/26 03:22:15.097\n \"ts\"=\"2026-04-09 03:22:15.097474\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\n STEP: creating a storage account for shoebox logs @ 04/09/26 03:40:30.261\n \"ts\"=\"2026-04-09 03:40:50.138070\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxzk56nx\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-6bnbg7/providers/Microsoft.Storage/storageAccounts/shoeboxzk56nx\"\n STEP: enabling diagnostic settings on the HCP cluster @ 04/09/26 03:40:50.138\n \"ts\"=\"2026-04-09 03:41:12.296341\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n STEP: waiting for log containers to appear in the storage account @ 04/09/26 03:41:12.296\n \"ts\"=\"2026-04-09 04:02:12.501062\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxzk56nx\" \"timestamp\"=\"2026-04-09T04:02:12Z\"\n \"ts\"=\"2026-04-09 04:02:12.501238\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 04:02:12.501321\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-5c55gc\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\n \"ts\"=\"2026-04-09 04:02:12.656846\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-5c55gc\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 04:02:13.973758\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-09 04:02:13.973873\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-09 04:02:13.973934\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 04:02:13.975374\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-130\"\n \"ts\"=\"2026-04-09 04:02:13.975384\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\n \"ts\"=\"2026-04-09 04:02:14.127141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-130\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 04:02:14.132746\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n \"ts\"=\"2026-04-09 04:02:14.257408\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-130\" \"deployment\"=\"msi-deployment-130\"\n \"ts\"=\"2026-04-09 04:02:14.262704\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-5c55gc\"\n \"ts\"=\"2026-04-09 04:02:14.414204\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 04:02:14.414333\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 04:02:14.414469\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ns but only 0 are assigned\" \"attempt\"=1 \"retryIn\"=\"1m0s\" \"elapsed\"=\"0s\"\n \"ts\"=\"2026-04-09 03:16:11.746501\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=2 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1m0s\"\n \"ts\"=\"2026-04-09 03:17:11.749675\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=3 \"retryIn\"=\"1m0s\" \"elapsed\"=\"2m0s\"\n \"ts\"=\"2026-04-09 03:18:11.752116\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=4 \"retryIn\"=\"1m0s\" \"elapsed\"=\"3m0s\"\n \"ts\"=\"2026-04-09 03:19:11.754307\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=5 \"retryIn\"=\"1m0s\" \"elapsed\"=\"4m0s\"\n \"ts\"=\"2026-04-09 03:20:11.756347\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=6 \"retryIn\"=\"1m0s\" \"elapsed\"=\"5m0s\"\n \"ts\"=\"2026-04-09 03:21:11.766667\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=7 \"elapsed\"=\"6m0s\"\n STEP: creating a resource group @ 04/09/26 03:21:11.766\n \"ts\"=\"2026-04-09 03:21:11.766746\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\n STEP: creating cluster parameters @ 04/09/26 03:21:16.342\n STEP: creating customer resources @ 04/09/26 03:21:16.342\n STEP: creating the HCP cluster @ 04/09/26 03:22:15.097\n \"ts\"=\"2026-04-09 03:22:15.097474\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\n STEP: creating a storage account for shoebox logs @ 04/09/26 03:40:30.261\n \"ts\"=\"2026-04-09 03:40:50.138070\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxzk56nx\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-6bnbg7/providers/Microsoft.Storage/storageAccounts/shoeboxzk56nx\"\n STEP: enabling diagnostic settings on the HCP cluster @ 04/09/26 03:40:50.138\n \"ts\"=\"2026-04-09 03:41:12.296341\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n STEP: waiting for log containers to appear in the storage account @ 04/09/26 03:41:12.296\n \"ts\"=\"2026-04-09 04:02:12.501062\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxzk56nx\" \"timestamp\"=\"2026-04-09T04:02:12Z\"\n \"ts\"=\"2026-04-09 04:02:12.501238\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-09 04:02:12.501321\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-5c55gc\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\n \"ts\"=\"2026-04-09 04:02:12.656846\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-5c55gc\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-09 04:02:13.973758\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-09 04:02:13.973873\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-09 04:02:13.973934\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-09 04:02:13.975374\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-130\"\n \"ts\"=\"2026-04-09 04:02:13.975384\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\n \"ts\"=\"2026-04-09 04:02:14.127141\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-130\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-09 04:02:14.132746\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n \"ts\"=\"2026-04-09 04:02:14.257408\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-130\" \"deployment\"=\"msi-deployment-130\"\n \"ts\"=\"2026-04-09 04:02:14.262704\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-5c55gc\"\n \"ts\"=\"2026-04-09 04:02:14.414204\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-09 04:02:14.414333\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-09 04:02:14.414469\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-6bnbg7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"fa5d5e232f6043e76cf619cf8a4d807dbaf6fd4985e853ef7ab69e4ec04cacb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","failed_at":"e2e","signature_id":"01edd05054d3d9450ab30aa909069d2d80dc296ce304743832a27fede9a35a05","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b03170\u003e: \n failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000b03170\u003e: \n failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2rlk9m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-2rlk9m, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","635a2afde9e3afd24ee6abfeb373ae222c4b51e9e9a5fed30800c3f45ff07db5","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}