{"meta":{"date":"2026-04-10","timezone":"UTC","generated_at":"2026-08-30T09:27:29Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":11,"failed_runs":9,"runs_with_occurrences":9,"runs_with_failure_pattern_matches":9,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T20:29:03Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"053152c9d54fff9f27f2f994d74445b3a235d008e63d0d50646e586bb1d02b75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","failed_at":"e2e","signature_id":"324ffdaba18d3147faf9a4285ad6fa60df88e7720fcd0b876adf23da5ffa8578","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007fd2f0\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mhtwts\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-mhtwts, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007fd2f0\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mhtwts\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-mhtwts, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"406f784c1979a8d942bedebb141c8a0e0b017b22fbbec8c9a0dcab4efa69e744","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","failed_at":"e2e","signature_id":"ad1a784b6233c088bf1cbcdab05052ce4206e83faaf04a30cff15ea5dc427b03","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfc0f0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-20-vl26hx\" in resourcegroup=\"rg-candidate-4-20-lbfbj4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-20-vl26hx in resource group rg-candidate-4-20-lbfbj4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dfc0f0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-20-vl26hx\" in resourcegroup=\"rg-candidate-4-20-lbfbj4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-20-vl26hx in resource group rg-candidate-4-20-lbfbj4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"4fa8618bdbc9ba6676da9b1fc4375c1a85a41f86ea0c7f5a2b32089fbb62d96a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","failed_at":"e2e","signature_id":"f9d8e75397a206a6268f651b0be6637d057a861d0bbf464e067b67c82a1421dd","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015c05a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cw77vd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cw77vd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015c05a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cw77vd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cw77vd, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"536ffeb9eb6a2c1e6ce71edce8da5ecf81781c188636b1ef4b7b7f738183c2a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","failed_at":"e2e","signature_id":"f4a2a227d30c0371e3cd0edc206bb57bb934f7573ea663644af9bd1f992bdc4f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00156eb10\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-6mnfl6, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6mnfl6 in resource group rg-zstream-upgrade-4-19-bjj5n4, error: failed waiting for cluster=\"cluster-zstream-4-19-6mnfl6\" in resourcegroup=\"rg-zstream-upgrade-4-19-bjj5n4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6mnfl6 in resource group rg-zstream-upgrade-4-19-bjj5n4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00156eb10\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-6mnfl6, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6mnfl6 in resource group rg-zstream-upgrade-4-19-bjj5n4, error: failed waiting for cluster=\"cluster-zstream-4-19-6mnfl6\" in resourcegroup=\"rg-zstream-upgrade-4-19-bjj5n4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6mnfl6 in resource group rg-zstream-upgrade-4-19-bjj5n4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"7cbd566ef8c58ce9eb038ee313428348f3f064b9c9c6a9b546055ed83ce6bac0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","failed_at":"e2e","signature_id":"55ef6ef4d9ea61f11d4d23769864db3f0d507685dc0ef554cb5890b869bcca8a","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:239]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c3a700\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9hc929/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:239]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c3a700\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9hc929/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"82a7756b322967262555cda3d795b25a94a325306ed17c8d2eeea76e306edacf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","failed_at":"e2e","signature_id":"2e4c68add548aaa6737eadeda2aefb454957e84c997a068f440595794fe5ea8b","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e73080\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e73080\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"fad9b4e6c5653eb44badf03470ef2ea7057edd0e169db024a39e12e2dd31c385","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","failed_at":"e2e","signature_id":"4417f9ebee113ede569a63481c296343013ef4c0dd63cb13ca927a1c9d0575a6","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:116]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012c4a20\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-2xwn4s\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-2xwn4s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:116]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012c4a20\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-2xwn4s\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-2xwn4s, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042684774031560704","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T19:22:53Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fef4276cf0842f27e6d31da743908d933544b363531a5ef747affb309dc66c9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042684774031560704","occurred_at":"2026-04-10T19:22:53Z","failed_at":"e2e","signature_id":"a9d92a762e487ff6e17f5d7d33bf21ab33f872b713827d1508be3bd3f1e9a462","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095d2c0\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-ql4p2r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-ql4p2r, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095d2c0\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-ql4p2r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-ql4p2r, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T16:55:28Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"19d4e3ef493f175546846a6c3483629089ed79eaa142a012a1c367d4ab10bd4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"a8c8fd1a5a8ee69127b99cd5468e4422b9f95af227abcd6b8f9e4da53d12eddf","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00092c4b0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-2rbk85, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-2rbk85 in resource group rg-zstream-upgrade-4-21-zqf4kq, error: failed waiting for cluster=\"cluster-zstream-4-21-2rbk85\" in resourcegroup=\"rg-zstream-upgrade-4-21-zqf4kq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-2rbk85 in resource group rg-zstream-upgrade-4-21-zqf4kq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00092c4b0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-2rbk85, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-2rbk85 in resource group rg-zstream-upgrade-4-21-zqf4kq, error: failed waiting for cluster=\"cluster-zstream-4-21-2rbk85\" in resourcegroup=\"rg-zstream-upgrade-4-21-zqf4kq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-2rbk85 in resource group rg-zstream-upgrade-4-21-zqf4kq, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"4e829be83fef5dc0aeb690403a8a820cec3c8e846fa2a7b2f87dcea05c5aeae0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"3c74c2b7b0d98970008636e4702e3dafe55cd39f4902e823fddd1b1caeaf1437","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009d5ec0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-bnsj87, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bnsj87 in resource group rg-np-version-upgrade-bnsj87-27s69m, error: failed waiting for cluster=\"np-version-upgrade-cluster-bnsj87\" in resourcegroup=\"rg-np-version-upgrade-bnsj87-27s69m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bnsj87 in resource group rg-np-version-upgrade-bnsj87-27s69m, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009d5ec0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-bnsj87, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bnsj87 in resource group rg-np-version-upgrade-bnsj87-27s69m, error: failed waiting for cluster=\"np-version-upgrade-cluster-bnsj87\" in resourcegroup=\"rg-np-version-upgrade-bnsj87-27s69m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-bnsj87 in resource group rg-np-version-upgrade-bnsj87-27s69m, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"627fb5b6ffcef6c799de0398ed85870e960fb79db7873bf7acae5285c1946d81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"9c9530d101e12ab5a650c8fc6e9715606e353ecc95c0653e2075234a94e464fa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015240c0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-20-7bz6dn\" in resourcegroup=\"rg-candidate-4-20-bwpkkz\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-20-7bz6dn in resource group rg-candidate-4-20-bwpkkz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015240c0\u003e: \n    failed waiting for hcpCluster=\"cluster-candidate-4-20-7bz6dn\" in resourcegroup=\"rg-candidate-4-20-bwpkkz\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-20-7bz6dn in resource group rg-candidate-4-20-bwpkkz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"74672be470a344fea62400bab2232f487d8723c3d8d03ee10b86062f936578ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"e1e9672e98e7c5f28b1706f8fa646f2740d0999ee129fbdcef9034e7e05cce00","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009241e0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-19-pxx6tf\" in resourcegroup=\"rg-zstream-upgrade-4-19-t99kbg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-pxx6tf in resource group rg-zstream-upgrade-4-19-t99kbg, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009241e0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-19-pxx6tf\" in resourcegroup=\"rg-zstream-upgrade-4-19-t99kbg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-pxx6tf in resource group rg-zstream-upgrade-4-19-t99kbg, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"78a7c3e64f1bebb16621ed3866de506a41b8c937c9cb6de1c7d948b6ca781f10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"0d0065a85884f57dc55146b3cd6f9e7f652f0498eab4c9f7cd357e374c69da98","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000985290\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-b54j8c, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-b54j8c\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-b54j8c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000985290\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-b54j8c, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-b54j8c\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-b54j8c, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"814496c110336eb6003c5dfc4da0c99e5439b811fa5b5080faa9a657bf086fc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"7a0ad282eff611158e8e43418bc514839cbe0023046786a1ce9aab3f8e0a2e7a","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b774d0\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-7sl9xl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-7sl9xl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b774d0\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-7sl9xl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-7sl9xl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"8cd8546b718c1c15a7c065e48efc65ec7d84bd5cc28c145f136f3c8a08bdccf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"3acbfe85d42d4b2af0028ce47dd46b2ecf6328bee6e74b263673d44b4664b09c","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009823c0\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-vtr9x4, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-vtr9x4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-vtr9x4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009823c0\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-vtr9x4, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-vtr9x4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-vtr9x4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"8eb9239ff66537e3b50dd80153da1824680346b6216bb18268744d912944fe9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"3342dc667feb6683fa8d37b7adb8bf7cebbf78de37e827acc0e3826580e28e1e","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015640c0\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-xh7g84, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xh7g84\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-xh7g84, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015640c0\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-xh7g84, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xh7g84\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-xh7g84, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"90de2ab4854d1a6b6fbaf54eddd024fadc0fca5f0c9f98a97e51c7ca856ea50f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"97ed143ef45d8dc14377b681c99ace8f580e81f564813b645e8377e8d924dee7","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ed8990\u003e: \n    failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-26vrlj\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-26vrlj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ed8990\u003e: \n    failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-26vrlj\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-26vrlj, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"95f046e4ac83dc364febc85be72e54c9c00392a1c16cabe1e7ba926fd0a2aef3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"5a88ff7c5b00148eaccf0a7f4dea26b63c8ec41eaceae91a95ba6b5e42525ddc","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010ffb60\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-6z9xfq\" in resourcegroup=\"admin-credential-lifecycle-test-nzvvl9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-6z9xfq in resource group admin-credential-lifecycle-test-nzvvl9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010ffb60\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-6z9xfq\" in resourcegroup=\"admin-credential-lifecycle-test-nzvvl9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-6z9xfq in resource group admin-credential-lifecycle-test-nzvvl9, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"d0268aab9bddf1c03b13c79eded331ec7b480b3aff12a24626bd24857d577e97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"466a1bc425e7596e811a1a5b201d5b80622e0baf5fe5f127acd6d33a30be9d6a","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008df680\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-mrrj8n, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-mrrj8n\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-mrrj8n, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008df680\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-mrrj8n, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-mrrj8n\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-mrrj8n, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"e747615fc6b5e1d69dd9510fe888c99093ec7d977d731d61823e09e66c495d9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"7c5da14974f842e2f7863d15e9875fdc09997be97e160621aa4a3cd853a7b438","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013de780\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-whzfsl, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-whzfsl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-whzfsl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013de780\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-whzfsl, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-whzfsl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-whzfsl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"f63970a89857208611ff324abe0ce07fdf849c54768c3c905fb290945598df89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","failed_at":"e2e","signature_id":"094a6c2314265f60b76e8fcbe84a2c8e44259e101a52c5a4c142e5713aec8804","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bda7b0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-20-wq2lcq\" in resourcegroup=\"rg-zstream-upgrade-4-20-nspgt8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-wq2lcq in resource group rg-zstream-upgrade-4-20-nspgt8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bda7b0\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-20-wq2lcq\" in resourcegroup=\"rg-zstream-upgrade-4-20-nspgt8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-wq2lcq in resource group rg-zstream-upgrade-4-20-nspgt8, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T16:55:01Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"04c8c760213ce9a5645b29914a1c77f31102c5644180e54ea8c88ec7e1ac65d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"e09f5da2799329b5ee78a500dfa49b480e4ee88c92315c4c097a19512e99ec74","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009b6280\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-tflxwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009b6280\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-tflxwt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"06a2e2f606428ebd3612de21dae75f081551dc600490c318bfce59eb380a9945","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"1b2c3288fde7d0baf2d757029c5c1cfe45b7c15828e1e4ac53ffd4d897080b18","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e90120\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-rn26mb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e90120\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-rn26mb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2b4fc1e3e601b9504098cfe4c55df5efbc0b392c461b701da555fa0666e10e30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"633be0bfad8cef28de804331d1e9aee9b444f7683897f6917a5919e6188f7c6c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001442240\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-w6h4l7, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-w6h4l7 in resource group rg-zstream-upgrade-4-21-dcfn2j, error: failed waiting for cluster=\"cluster-zstream-4-21-w6h4l7\" in resourcegroup=\"rg-zstream-upgrade-4-21-dcfn2j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-w6h4l7 in resource group rg-zstream-upgrade-4-21-dcfn2j, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001442240\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-w6h4l7, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-w6h4l7 in resource group rg-zstream-upgrade-4-21-dcfn2j, error: failed waiting for cluster=\"cluster-zstream-4-21-w6h4l7\" in resourcegroup=\"rg-zstream-upgrade-4-21-dcfn2j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-w6h4l7 in resource group rg-zstream-upgrade-4-21-dcfn2j, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"33ad32cb4c2502b313b25f9c52a7843565e72aacd78510191bc43fbb967e1b03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"d4beb9840975059b78ebb2bbdc1f3bd005e206d72191c83051709934b2acc299","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ae0210\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-fc89tz, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-fc89tz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-fc89tz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ae0210\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-fc89tz, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-fc89tz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-fc89tz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"4a87f45ef9f50d6fe831bb2870b38d86df0fb8005f69cb83388a545dedc4b6fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"ef4ac3a0886e7bfd19c86141172c407d907b090c56c349762c5baab5d47ea58a","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:117]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dcec00\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-8w8qgx, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8w8qgx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-8w8qgx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:117]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dcec00\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-8w8qgx, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8w8qgx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-8w8qgx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"6f832959e59b5c8c41dbfbe426f535c5f30a4a3c9b255691c8dc21f9b2690f33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"0591505a8d0d09ee27d773b578770c0879cc0fd3c1615c87b0b878047697e942","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015a00f0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-dgwgxd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-dgwgxd in resource group rg-zstream-upgrade-4-20-5mx8l9, error: failed waiting for cluster=\"cluster-zstream-4-20-dgwgxd\" in resourcegroup=\"rg-zstream-upgrade-4-20-5mx8l9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-dgwgxd in resource group rg-zstream-upgrade-4-20-5mx8l9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015a00f0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-dgwgxd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-dgwgxd in resource group rg-zstream-upgrade-4-20-5mx8l9, error: failed waiting for cluster=\"cluster-zstream-4-20-dgwgxd\" in resourcegroup=\"rg-zstream-upgrade-4-20-5mx8l9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-dgwgxd in resource group rg-zstream-upgrade-4-20-5mx8l9, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"b1cbebb7547a17f4c0e52242a48d94f2ea82de51104e1f88c04306a72cb0f7f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"685faef7c56cc1e714facd7df48e1f0acf7fc3632e9fd34b791af4ada2983c56","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e6eb10\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e6eb10\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"b1fe1b3908b76e4bea2f2ec687b4606cfe1d352fef723b8b748dc6501b22f394","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"f726f5cdc0a97832459d0a83f77b5fc8d60caabf1f5995ed8b497222589a4f3d","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00053ad20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-vf4zmt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-rd4xqc/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00053ad20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-vf4zmt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-rd4xqc/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ed31f024ba7078d911897ed7e581b69eeb5596e92d60258bebbd96a2171344ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"79e17694bc7eb6852bb1fb17ca10ba1644573ee030a39c7db08020c8c4393908","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010c1a40\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-cbtf79, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-cbtf79\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-cbtf79, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010c1a40\u003e: \n    failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-cbtf79, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-cbtf79\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-cbtf79, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"f4ef2d00786503bd99411eddc3d00fa72d26c3466e00d805bc1727f9b4aff143","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","failed_at":"e2e","signature_id":"3f552bc1f85dd1b4bd1189c16824c26c1b464f002c113d93d56f72f56c0dcc27","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001312300\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-26bkkw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-phh9x4/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001312300\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-26bkkw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-phh9x4/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T10:59:58Z"},"failed_at":["e2e","unknown"],"failed_test_count":5,"occurrences":[{"row_id":"19d48593694af6938bfd91844258591bcecadf583681b2de6812cbfd95ec7966","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T13:03:33Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"3bd4b5af9f891ee7ca3199e2684f9120a36f12d94d1f577b99c5e9c3c6919737","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","failed_at":"e2e","signature_id":"e115f0d91ca258d05c604dcc10d976dfb30b4ee3c98378e747983d1d19c90761","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"t/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\nW0410 12:04:14.665210     893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 12:04:14.748424     893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 12:04:14.748740\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:893\"\n  \"ts\"=\"2026-04-10 12:04:14.933427\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 12:04:14.933\n  \"ts\"=\"2026-04-10 12:04:14.933646\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 12:04:19.473\n    STEP: creating customer resources @ 04/10/26 12:04:19.579\n    STEP: creating the HCP cluster with version 4.20.17 @ 04/10/26 12:05:04.201\n  \"ts\"=\"2026-04-10 12:05:04.201160\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-jnm8s9\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n    STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:16:02.227\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:01.442\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:09.867\n    STEP: capturing node release images before upgrade @ 04/10/26 12:29:10.041\n    STEP: triggering nodepool upgrade to version 4.20.17 @ 04/10/26 12:29:10.08\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:52.504\n    STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:54.086\n  \"ts\"=\"2026-04-10 12:51:54.286433\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:51:54.286512\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n  \"ts\"=\"2026-04-10 12:51:54.379693\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:51:55.257214\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:51:55.257316\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:51:55.257391\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:51:55.259038\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\"\n  \"ts\"=\"2026-04-10 12:51:55.259056\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n  \"ts\"=\"2026-04-10 12:51:55.353380\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"pooledNonMsiScopedAssignments-jfl6npesxtst4\"\n  \"ts\"=\"2026-04-10 12:51:55.415329\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\"\n  \"ts\"=\"2026-04-10 12:51:55.578837\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:51:55.709848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"msi-deployment-129\"\n  \"ts\"=\"2026-04-10 12:51:55.860303\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:51:55.860430\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:51:55.860601\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n  \"ts\"=\"2026-04-10 13:02:14.746040\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n  \"ts\"=\"2026-04-10 13:02:14.746065\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\nW0410 12:04:14.665210     893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 12:04:14.748424     893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 12:04:14.748740\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:893\"\n  \"ts\"=\"2026-04-10 12:04:14.933427\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 12:04:14.933\n  \"ts\"=\"2026-04-10 12:04:14.933646\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 12:04:19.473\n    STEP: creating customer resources @ 04/10/26 12:04:19.579\n    STEP: creating the HCP cluster with version 4.20.17 @ 04/10/26 12:05:04.201\n  \"ts\"=\"2026-04-10 12:05:04.201160\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-jnm8s9\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n    STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:16:02.227\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:01.442\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:09.867\n    STEP: capturing node release images before upgrade @ 04/10/26 12:29:10.041\n    STEP: triggering nodepool upgrade to version 4.20.17 @ 04/10/26 12:29:10.08\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:52.504\n    STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:54.086\n  \"ts\"=\"2026-04-10 12:51:54.286433\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:51:54.286512\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n  \"ts\"=\"2026-04-10 12:51:54.379693\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:51:55.257214\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:51:55.257316\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:51:55.257391\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:51:55.259038\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\"\n  \"ts\"=\"2026-04-10 12:51:55.259056\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n  \"ts\"=\"2026-04-10 12:51:55.353380\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"pooledNonMsiScopedAssignments-jfl6npesxtst4\"\n  \"ts\"=\"2026-04-10 12:51:55.415329\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\"\n  \"ts\"=\"2026-04-10 12:51:55.578837\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:51:55.709848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"msi-deployment-129\"\n  \"ts\"=\"2026-04-10 12:51:55.860303\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:51:55.860430\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:51:55.860601\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n  \"ts\"=\"2026-04-10 13:02:14.746040\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n  \"ts\"=\"2026-04-10 13:02:14.746065\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"48f3ef72baefeb866af53223094318020806a9272432e2a3c7e2743f27c49929","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","failed_at":"e2e","signature_id":"7ba7fc6de5bcafc9ea168179205860c649c276fe4162534ff6f785c88ed1a1bf","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775823245 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n  \"ts\"=\"2026-04-10 12:14:05.070742\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:919\"\n  \"ts\"=\"2026-04-10 12:14:05.079100\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 12:14:05.079\n  \"ts\"=\"2026-04-10 12:14:05.079151\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n    STEP: creating cluster parameters @ 04/10/26 12:14:09.56\n    STEP: creating customer resources @ 04/10/26 12:14:09.56\n    STEP: creating the HCP cluster @ 04/10/26 12:14:54.459\n  \"ts\"=\"2026-04-10 12:14:54.459322\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n    STEP: creating a storage account for shoebox logs @ 04/10/26 12:29:05.918\n  \"ts\"=\"2026-04-10 12:29:25.513737\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxfpqdwk\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7pqpv2/providers/Microsoft.Storage/storageAccounts/shoeboxfpqdwk\"\n    STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 12:29:25.513\n  \"ts\"=\"2026-04-10 12:29:30.702291\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n    STEP: waiting for log containers to appear in the storage account @ 04/10/26 12:29:30.702\n  \"ts\"=\"2026-04-10 12:55:30.837118\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxfpqdwk\" \"timestamp\"=\"2026-04-10T12:55:30Z\"\n  \"ts\"=\"2026-04-10 12:55:30.837292\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:55:30.837387\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n  \"ts\"=\"2026-04-10 12:55:30.988434\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:55:32.339790\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:55:32.339933\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:55:32.340053\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:55:32.341960\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\"\n  \"ts\"=\"2026-04-10 12:55:32.342015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n  \"ts\"=\"2026-04-10 12:55:32.680479\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-10 12:55:32.749543\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\"\n  \"ts\"=\"2026-04-10 12:55:32.952168\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:55:33.567562\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"msi-deployment-95\"\n  \"ts\"=\"2026-04-10 12:55:33.845873\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:55:33.846012\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:55:33.846126\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775823245 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n  \"ts\"=\"2026-04-10 12:14:05.070742\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:919\"\n  \"ts\"=\"2026-04-10 12:14:05.079100\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 12:14:05.079\n  \"ts\"=\"2026-04-10 12:14:05.079151\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n    STEP: creating cluster parameters @ 04/10/26 12:14:09.56\n    STEP: creating customer resources @ 04/10/26 12:14:09.56\n    STEP: creating the HCP cluster @ 04/10/26 12:14:54.459\n  \"ts\"=\"2026-04-10 12:14:54.459322\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n    STEP: creating a storage account for shoebox logs @ 04/10/26 12:29:05.918\n  \"ts\"=\"2026-04-10 12:29:25.513737\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxfpqdwk\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7pqpv2/providers/Microsoft.Storage/storageAccounts/shoeboxfpqdwk\"\n    STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 12:29:25.513\n  \"ts\"=\"2026-04-10 12:29:30.702291\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n    STEP: waiting for log containers to appear in the storage account @ 04/10/26 12:29:30.702\n  \"ts\"=\"2026-04-10 12:55:30.837118\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxfpqdwk\" \"timestamp\"=\"2026-04-10T12:55:30Z\"\n  \"ts\"=\"2026-04-10 12:55:30.837292\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:55:30.837387\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n  \"ts\"=\"2026-04-10 12:55:30.988434\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:55:32.339790\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:55:32.339933\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:55:32.340053\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:55:32.341960\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\"\n  \"ts\"=\"2026-04-10 12:55:32.342015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n  \"ts\"=\"2026-04-10 12:55:32.680479\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-10 12:55:32.749543\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\"\n  \"ts\"=\"2026-04-10 12:55:32.952168\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:55:33.567562\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"msi-deployment-95\"\n  \"ts\"=\"2026-04-10 12:55:33.845873\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:55:33.846012\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:55:33.846126\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"b717e13db12bfc39fc084cd7d9a164a7ed7920fdec49fa77ec8d01a228c2ae94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","failed_at":"e2e","signature_id":"1656f2b598b891e3f588923f365c70c85887da29099479a61fa374a63b758405","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 12:03:04.020611\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:880\"\n  \"ts\"=\"2026-04-10 12:03:04.028462\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 12:03:04.028\n  \"ts\"=\"2026-04-10 12:03:04.028545\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 12:03:08.089\nW0410 12:03:08.168105     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: creating customer resources @ 04/10/26 12:03:08.168\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 12:04:03.752\n  \"ts\"=\"2026-04-10 12:04:03.752222\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-wdq4gs\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:15:44.21\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:06.745\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:05.032\nW0410 12:29:05.158022     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: capturing node release images before upgrade @ 04/10/26 12:29:05.158\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 12:29:05.194\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:48.069\n    STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:49.726\n  \"ts\"=\"2026-04-10 12:51:50.360556\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:51:50.360655\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n  \"ts\"=\"2026-04-10 12:51:50.487211\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:51:51.762518\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:51:51.762617\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:51:51.762676\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:51:51.764204\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\"\n  \"ts\"=\"2026-04-10 12:51:51.764225\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n  \"ts\"=\"2026-04-10 12:51:51.859754\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"pooledNonMsiScopedAssignments-dpbn3ikyauobc\"\n  \"ts\"=\"2026-04-10 12:51:52.188516\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\"\n  \"ts\"=\"2026-04-10 12:51:52.242487\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:51:52.779918\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"msi-deployment-131\"\n  \"ts\"=\"2026-04-10 12:51:52.917289\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:51:52.917406\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:51:52.917517\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n  \"ts\"=\"2026-04-10 13:02:10.173344\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n  \"ts\"=\"2026-04-10 13:02:10.173371\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nAPI server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 12:03:04.020611\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:880\"\n  \"ts\"=\"2026-04-10 12:03:04.028462\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 12:03:04.028\n  \"ts\"=\"2026-04-10 12:03:04.028545\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 12:03:08.089\nW0410 12:03:08.168105     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: creating customer resources @ 04/10/26 12:03:08.168\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 12:04:03.752\n  \"ts\"=\"2026-04-10 12:04:03.752222\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-wdq4gs\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:15:44.21\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:06.745\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:05.032\nW0410 12:29:05.158022     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: capturing node release images before upgrade @ 04/10/26 12:29:05.158\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 12:29:05.194\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:48.069\n    STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:49.726\n  \"ts\"=\"2026-04-10 12:51:50.360556\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:51:50.360655\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n  \"ts\"=\"2026-04-10 12:51:50.487211\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:51:51.762518\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:51:51.762617\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:51:51.762676\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:51:51.764204\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\"\n  \"ts\"=\"2026-04-10 12:51:51.764225\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n  \"ts\"=\"2026-04-10 12:51:51.859754\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"pooledNonMsiScopedAssignments-dpbn3ikyauobc\"\n  \"ts\"=\"2026-04-10 12:51:52.188516\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\"\n  \"ts\"=\"2026-04-10 12:51:52.242487\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:51:52.779918\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"msi-deployment-131\"\n  \"ts\"=\"2026-04-10 12:51:52.917289\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:51:52.917406\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:51:52.917517\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n  \"ts\"=\"2026-04-10 13:02:10.173344\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n  \"ts\"=\"2026-04-10 13:02:10.173371\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"df211064f12efc10cdfec45b0d345c9dac81d76256b870eafceff0beb973a938","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","failed_at":"e2e","signature_id":"71b0ec01edd9207bb394cba62c96fa66ee1e1f320a74672b8a730144deb83883","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:294]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001508c80\u003e: \n    failed creating external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-n5j64n\" for cluster=\"cidr-connectivity-test\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n5j64n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/externalAuths/ext-auth-cidr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Cluster '2pj8n2ov52hfcbj1lfepdkk77i8o48uj' has an external authentication pending deletion\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:294]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001508c80\u003e: \n    failed creating external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-n5j64n\" for cluster=\"cidr-connectivity-test\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n5j64n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/externalAuths/ext-auth-cidr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Cluster '2pj8n2ov52hfcbj1lfepdkk77i8o48uj' has an external authentication pending deletion\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4acfe441005f8003e7187d8bb50b5a3c1645ce7924b7f2de03ef6bdb2cb05e18","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Cluster '\u003cid\u003e' has an external authentication pending deletion; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["4acfe441005f8003e7187d8bb50b5a3c1645ce7924b7f2de03ef6bdb2cb05e18","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T10:57:56Z"},"failed_at":["e2e","unknown"],"failed_test_count":9,"occurrences":[{"row_id":"08addc33a2c02af55f4a289afa96df00dc7ace37ecdfe614387c6dd322b4208e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"e2e","signature_id":"e2072d0c469b46a1d7f7d3e238ea467288fa9ef089199ab06d9636c4c900cdef","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775823200 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n  \"ts\"=\"2026-04-10 12:13:20.678047\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:920\"\n  \"ts\"=\"2026-04-10 12:13:20.684932\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 12:13:20.684\n  \"ts\"=\"2026-04-10 12:13:20.685003\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\n    STEP: creating cluster parameters @ 04/10/26 12:13:25.393\n    STEP: creating customer resources @ 04/10/26 12:13:25.393\n    STEP: creating the HCP cluster @ 04/10/26 12:14:15.839\n  \"ts\"=\"2026-04-10 12:14:15.840100\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\n    STEP: creating a storage account for shoebox logs @ 04/10/26 12:26:52.987\n  \"ts\"=\"2026-04-10 12:27:14.860715\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxcv42l9\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cgrnxm/providers/Microsoft.Storage/storageAccounts/shoeboxcv42l9\"\n    STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 12:27:14.86\n  \"ts\"=\"2026-04-10 12:27:16.973481\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n    STEP: waiting for log containers to appear in the storage account @ 04/10/26 12:27:16.973\n  \"ts\"=\"2026-04-10 12:56:17.519891\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxcv42l9\" \"timestamp\"=\"2026-04-10T12:56:17Z\"\n  \"ts\"=\"2026-04-10 12:56:17.520097\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:56:17.520196\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-kcfn5p\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\n  \"ts\"=\"2026-04-10 12:56:18.474282\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-kcfn5p\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:56:19.927127\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:56:19.927234\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:56:19.927293\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:56:19.928736\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-138\"\n  \"ts\"=\"2026-04-10 12:56:19.928752\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\n  \"ts\"=\"2026-04-10 12:56:20.089477\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-138\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:56:20.632817\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-138\" \"deployment\"=\"msi-deployment-138\"\n  \"ts\"=\"2026-04-10 12:56:20.686097\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-10 12:56:20.911100\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-kcfn5p\"\n  \"ts\"=\"2026-04-10 12:56:21.092180\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:56:21.092317\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:56:21.092433\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775823200 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n  \"ts\"=\"2026-04-10 12:13:20.678047\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:920\"\n  \"ts\"=\"2026-04-10 12:13:20.684932\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 12:13:20.684\n  \"ts\"=\"2026-04-10 12:13:20.685003\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\n    STEP: creating cluster parameters @ 04/10/26 12:13:25.393\n    STEP: creating customer resources @ 04/10/26 12:13:25.393\n    STEP: creating the HCP cluster @ 04/10/26 12:14:15.839\n  \"ts\"=\"2026-04-10 12:14:15.840100\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\n    STEP: creating a storage account for shoebox logs @ 04/10/26 12:26:52.987\n  \"ts\"=\"2026-04-10 12:27:14.860715\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxcv42l9\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cgrnxm/providers/Microsoft.Storage/storageAccounts/shoeboxcv42l9\"\n    STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 12:27:14.86\n  \"ts\"=\"2026-04-10 12:27:16.973481\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n    STEP: waiting for log containers to appear in the storage account @ 04/10/26 12:27:16.973\n  \"ts\"=\"2026-04-10 12:56:17.519891\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxcv42l9\" \"timestamp\"=\"2026-04-10T12:56:17Z\"\n  \"ts\"=\"2026-04-10 12:56:17.520097\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:56:17.520196\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-kcfn5p\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\n  \"ts\"=\"2026-04-10 12:56:18.474282\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-kcfn5p\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:56:19.927127\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:56:19.927234\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:56:19.927293\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:56:19.928736\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-138\"\n  \"ts\"=\"2026-04-10 12:56:19.928752\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\n  \"ts\"=\"2026-04-10 12:56:20.089477\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-138\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:56:20.632817\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-138\" \"deployment\"=\"msi-deployment-138\"\n  \"ts\"=\"2026-04-10 12:56:20.686097\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-10 12:56:20.911100\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-kcfn5p\"\n  \"ts\"=\"2026-04-10 12:56:21.092180\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:56:21.092317\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:56:21.092433\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-cgrnxm\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"1e02639624563e8ba33f1a4bc79235c3ef8ca9162a6f4899d5e345cebc06f1b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"e2e","signature_id":"e72458e45371568f1114245b29ad35b4f5307ce068bdec27027c68de7680c7db","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000939b60\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-8bwmgr\" in resourcegroup=\"admin-credential-lifecycle-test-8h9b62\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-8bwmgr in resource group admin-credential-lifecycle-test-8h9b62, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000939b60\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-8bwmgr\" in resourcegroup=\"admin-credential-lifecycle-test-8h9b62\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-8bwmgr in resource group admin-credential-lifecycle-test-8h9b62, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"207610d3f84dcf668f23aa07e38a2b36e23ea6a819163ab97017e406f12a1916","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"e2e","signature_id":"a87063a43d794d60b0e265445eae9710463acd30489634d881a98f9be587a1a9","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001916420\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-19-mlfmcg\" in resourcegroup=\"rg-zstream-upgrade-4-19-7ks759\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-mlfmcg in resource group rg-zstream-upgrade-4-19-7ks759, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001916420\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-19-mlfmcg\" in resourcegroup=\"rg-zstream-upgrade-4-19-7ks759\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-mlfmcg in resource group rg-zstream-upgrade-4-19-7ks759, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"5aa9dfb2d9be6abcb05a954ce5bc7d00d3656422e0b8600147f63af95be85ff9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"e2e","signature_id":"4b59ed1e7bc1c9174eb070f50a729e281f859e483dcfa086bc53ac0d3016c8bb","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"dRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: 201: must be less than or equal to 200\\\",\\n    \\\"target\\\": \\\"properties.replicas\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: creating a nodepool @ 04/10/26 12:35:29.989\n    STEP: attempting to update nodepool version to higher than cluster version @ 04/10/26 12:48:40.069\n  \"ts\"=\"2026-04-10 12:48:41.506501\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-r9tmjd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: attempting to update immutable platform profile fields @ 04/10/26 12:48:41.506\n  \"ts\"=\"2026-04-10 12:48:42.314595\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-r9tmjd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"4 errors occurred\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  \"ts\"=\"2026-04-10 12:48:42.593186\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:48:42.593298\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-t7cvxj\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\"\n  \"ts\"=\"2026-04-10 12:48:43.140137\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-t7cvxj\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:48:45.405483\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:48:45.405582\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:48:45.405650\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:48:45.407151\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-81\"\n  \"ts\"=\"2026-04-10 12:48:45.407168\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\"\n  \"ts\"=\"2026-04-10 12:48:45.585306\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-81\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:48:45.685982\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n  \"ts\"=\"2026-04-10 12:48:45.901268\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-81\" \"deployment\"=\"msi-deployment-81\"\n  \"ts\"=\"2026-04-10 12:48:46.030194\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\" \"deployment\"=\"customer-infra-negative-tests-cluster-t7cvxj\"\n  \"ts\"=\"2026-04-10 12:48:46.498826\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:48:46.498977\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:48:46.499112\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ndRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: 201: must be less than or equal to 200\\\",\\n    \\\"target\\\": \\\"properties.replicas\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: creating a nodepool @ 04/10/26 12:35:29.989\n    STEP: attempting to update nodepool version to higher than cluster version @ 04/10/26 12:48:40.069\n  \"ts\"=\"2026-04-10 12:48:41.506501\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-r9tmjd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: attempting to update immutable platform profile fields @ 04/10/26 12:48:41.506\n  \"ts\"=\"2026-04-10 12:48:42.314595\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-r9tmjd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"4 errors occurred\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  \"ts\"=\"2026-04-10 12:48:42.593186\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:48:42.593298\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-t7cvxj\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\"\n  \"ts\"=\"2026-04-10 12:48:43.140137\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-t7cvxj\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:48:45.405483\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:48:45.405582\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:48:45.405650\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:48:45.407151\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-81\"\n  \"ts\"=\"2026-04-10 12:48:45.407168\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\"\n  \"ts\"=\"2026-04-10 12:48:45.585306\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-81\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:48:45.685982\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n  \"ts\"=\"2026-04-10 12:48:45.901268\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-81\" \"deployment\"=\"msi-deployment-81\"\n  \"ts\"=\"2026-04-10 12:48:46.030194\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\" \"deployment\"=\"customer-infra-negative-tests-cluster-t7cvxj\"\n  \"ts\"=\"2026-04-10 12:48:46.498826\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:48:46.498977\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:48:46.499112\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-r9tmjd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"5b8d538c3f17a3bf3924d67db413cd4a6a05cf6462a545e4985217879eeb7702","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"e2e","signature_id":"e5e871a9a2c633cc721fe9ca827b4448e9e818af940c191290f0b5f0ee7e369f","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775822557 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:223\nW0410 12:02:37.604097     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 12:02:37.689386     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 12:02:37.689749\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:880\"\n  \"ts\"=\"2026-04-10 12:02:37.700441\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 12:02:37.7\n  \"ts\"=\"2026-04-10 12:02:37.700555\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gntxgv-z6ddmq\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 12:02:41.704\nW0410 12:02:41.803680     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: creating customer resources @ 04/10/26 12:02:41.803\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 12:03:52.723\n  \"ts\"=\"2026-04-10 12:03:52.723365\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gntxgv\" \"resourceGroup\"=\"rg-np-version-upgrade-gntxgv-z6ddmq\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:20:10.496\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:35:31.882\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:37:32.153\nW0410 12:37:32.309157     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: capturing node release images before upgrade @ 04/10/26 12:37:32.309\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 12:37:32.515\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:38:17.362\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775822557 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:223\nW0410 12:02:37.604097     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 12:02:37.689386     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 12:02:37.689749\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:880\"\n  \"ts\"=\"2026-04-10 12:02:37.700441\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 12:02:37.7\n  \"ts\"=\"2026-04-10 12:02:37.700555\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gntxgv-z6ddmq\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 12:02:41.704\nW0410 12:02:41.803680     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: creating customer resources @ 04/10/26 12:02:41.803\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 12:03:52.723\n  \"ts\"=\"2026-04-10 12:03:52.723365\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gntxgv\" \"resourceGroup\"=\"rg-np-version-upgrade-gntxgv-z6ddmq\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:20:10.496\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:35:31.882\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:37:32.153\nW0410 12:37:32.309157     880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: capturing node release images before upgrade @ 04/10/26 12:37:32.309\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 12:37:32.515\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:38:17.362\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"87609b5fed3c2b4da993152349bcdedfff648144b9b74e327eae609db5741ebe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"e2e","signature_id":"b052f95c7ac43877358d9b58edf1d1cf79d94d9b5351ee84ab0314fd489ecd4f","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"t.openshift.io/managed\"=\"true\" \"key2\"=\"value2\" \"kubernetes.io/os\"=\"linux\" \"node-role.kubernetes.io/worker\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/nodePool\"=\"s5o9c5a9h3i2v6w-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-7qxvc\" \"topology.kubernetes.io/region\"=\"westeurope\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n  \"ts\"=\"2026-04-10 12:54:12.444384\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-p4tm7\" \"labels\"={\"kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/hostname\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-p4tm7\" \"node-role.kubernetes.io/worker\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"westeurope\" \"failure-domain.beta.kubernetes.io/zone\"=\"2\" \"kubernetes.io/os\"=\"linux\" \"topology.kubernetes.io/zone\"=\"2\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"node.openshift.io/os_id\"=\"rhel\" \"hypershift.openshift.io/nodePool\"=\"s5o9c5a9h3i2v6w-np-1\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n  \"ts\"=\"2026-04-10 12:54:12.444422\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-x9fkb\" \"labels\"={\"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"s5o9c5a9h3i2v6w-np-1\" \"topology.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/hostname\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-x9fkb\" \"kubernetes.io/os\"=\"linux\" \"node-role.kubernetes.io/worker\"=\"\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/region\"=\"westeurope\" \"beta.kubernetes.io/arch\"=\"amd64\" \"key1\"=\"value1\" \"kubernetes.io/arch\"=\"amd64\" \"topology.disk.csi.azure.com/zone\"=\"\" \"beta.kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n    STEP: verifying original labels persist on 3 nodes @ 04/10/26 12:54:12.444\n    STEP: verifying original taints persist on 2 nodes @ 04/10/26 12:54:12.444\n  \"ts\"=\"2026-04-10 12:54:12.444639\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:54:12.444722\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-wgg6nm\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\"\n  \"ts\"=\"2026-04-10 12:54:12.819345\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-wgg6nm\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:54:14.809808\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:54:14.809931\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:54:14.810001\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:54:14.811654\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-79\"\n  \"ts\"=\"2026-04-10 12:54:14.811682\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\"\n  \"ts\"=\"2026-04-10 12:54:14.976571\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n  \"ts\"=\"2026-04-10 12:54:15.157403\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-79\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:54:15.468975\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-wgg6nm\"\n  \"ts\"=\"2026-04-10 12:54:15.516325\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-79\" \"deployment\"=\"msi-deployment-79\"\n  \"ts\"=\"2026-04-10 12:54:15.711640\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:54:15.711769\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:54:15.711872\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nt.openshift.io/managed\"=\"true\" \"key2\"=\"value2\" \"kubernetes.io/os\"=\"linux\" \"node-role.kubernetes.io/worker\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/nodePool\"=\"s5o9c5a9h3i2v6w-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-7qxvc\" \"topology.kubernetes.io/region\"=\"westeurope\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n  \"ts\"=\"2026-04-10 12:54:12.444384\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-p4tm7\" \"labels\"={\"kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/hostname\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-p4tm7\" \"node-role.kubernetes.io/worker\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"westeurope\" \"failure-domain.beta.kubernetes.io/zone\"=\"2\" \"kubernetes.io/os\"=\"linux\" \"topology.kubernetes.io/zone\"=\"2\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"node.openshift.io/os_id\"=\"rhel\" \"hypershift.openshift.io/nodePool\"=\"s5o9c5a9h3i2v6w-np-1\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n  \"ts\"=\"2026-04-10 12:54:12.444422\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-x9fkb\" \"labels\"={\"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"s5o9c5a9h3i2v6w-np-1\" \"topology.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/hostname\"=\"s5o9c5a9h3i2v6w-np-1-lx4bc-x9fkb\" \"kubernetes.io/os\"=\"linux\" \"node-role.kubernetes.io/worker\"=\"\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/region\"=\"westeurope\" \"beta.kubernetes.io/arch\"=\"amd64\" \"key1\"=\"value1\" \"kubernetes.io/arch\"=\"amd64\" \"topology.disk.csi.azure.com/zone\"=\"\" \"beta.kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n    STEP: verifying original labels persist on 3 nodes @ 04/10/26 12:54:12.444\n    STEP: verifying original taints persist on 2 nodes @ 04/10/26 12:54:12.444\n  \"ts\"=\"2026-04-10 12:54:12.444639\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:54:12.444722\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-wgg6nm\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\"\n  \"ts\"=\"2026-04-10 12:54:12.819345\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-wgg6nm\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:54:14.809808\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:54:14.809931\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:54:14.810001\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:54:14.811654\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-79\"\n  \"ts\"=\"2026-04-10 12:54:14.811682\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\"\n  \"ts\"=\"2026-04-10 12:54:14.976571\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n  \"ts\"=\"2026-04-10 12:54:15.157403\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-79\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:54:15.468975\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-wgg6nm\"\n  \"ts\"=\"2026-04-10 12:54:15.516325\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-79\" \"deployment\"=\"msi-deployment-79\"\n  \"ts\"=\"2026-04-10 12:54:15.711640\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:54:15.711769\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:54:15.711872\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-2qpnkd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"b21091d7d7acb71ef2d6cddbf0a38cc9709af82d5d6f15c619748f31efa80079","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T13:01:55Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"df42561f9e7ab94fa9f2e36c5be0283fa534bd131a6096597c0f9fb426f3cb63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"e2e","signature_id":"d24344800d201228d64db010e816a5e25cf74acb86cc4c5d05002723bf4f2156","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"11\n    STEP: creating customer infrastructure and managed identities for third cluster @ 04/10/26 12:51:13.197\n    STEP: attempting to create a third cluster using the same managed resource group as the second cluster @ 04/10/26 12:51:52.918\n  \"ts\"=\"2026-04-10 12:51:52.918883\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster3\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n    STEP: verifying that the managed resource group still exists @ 04/10/26 12:51:54.38\n    STEP: testing resource group-level cluster listing @ 04/10/26 12:51:54.734\n  \"ts\"=\"2026-04-10 12:51:55.069173\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:51:55.069279\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-m8xtvh\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n  \"ts\"=\"2026-04-10 12:51:55.240577\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-m8xtvh\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:51:57.252380\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-pg7qtn\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n  \"ts\"=\"2026-04-10 12:51:57.630267\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-pg7qtn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:51:59.685527\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster3-zvf5lf\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n  \"ts\"=\"2026-04-10 12:51:59.858444\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster3-zvf5lf\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:52:01.491661\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:52:01.491763\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:52:01.491828\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:52:01.493244\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-44\"\n  \"ts\"=\"2026-04-10 12:52:01.493255\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-25\"\n  \"ts\"=\"2026-04-10 12:52:01.493268\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n  \"ts\"=\"2026-04-10 12:52:01.493296\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\"\n  \"ts\"=\"2026-04-10 12:52:01.646619\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:52:01.653898\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-44\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:52:01.685532\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-25\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:52:01.724961\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"pooledNonMsiScopedAssignments-igpwm4vimfr3y\"\n  \"ts\"=\"2026-04-10 12:52:01.776009\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\" \"deployment\"=\"msi-deployment-108\"\n  \"ts\"=\"2026-04-10 12:52:01.783539\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-44\" \"deployment\"=\"msi-deployment-44\"\n  \"ts\"=\"2026-04-10 12:52:01.867495\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"customer-infra-basic-hcp-cluster3-zvf5lf\"\n  \"ts\"=\"2026-04-10 12:52:02.032517\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-04-10 12:52:02.036082\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-25\" \"deployment\"=\"msi-deployment-25\"\n  \"ts\"=\"2026-04-10 12:52:02.328126\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-pg7qtn\"\n  \"ts\"=\"2026-04-10 12:52:03.086364\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-04-10 12:52:03.725551\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"customer-infra-basic-hcp-cluster-m8xtvh\"\n  \"ts\"=\"2026-04-10 12:52:04.092347\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:52:04.092476\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:52:04.092575\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-l8rh58\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n11\n    STEP: creating customer infrastructure and managed identities for third cluster @ 04/10/26 12:51:13.197\n    STEP: attempting to create a third cluster using the same managed resource group as the second cluster @ 04/10/26 12:51:52.918\n  \"ts\"=\"2026-04-10 12:51:52.918883\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster3\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n    STEP: verifying that the managed resource group still exists @ 04/10/26 12:51:54.38\n    STEP: testing resource group-level cluster listing @ 04/10/26 12:51:54.734\n  \"ts\"=\"2026-04-10 12:51:55.069173\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 12:51:55.069279\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-m8xtvh\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n  \"ts\"=\"2026-04-10 12:51:55.240577\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-m8xtvh\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:51:57.252380\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-pg7qtn\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n  \"ts\"=\"2026-04-10 12:51:57.630267\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-pg7qtn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:51:59.685527\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster3-zvf5lf\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n  \"ts\"=\"2026-04-10 12:51:59.858444\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster3-zvf5lf\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 12:52:01.491661\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:52:01.491763\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-04-10 12:52:01.491828\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 12:52:01.493244\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-44\"\n  \"ts\"=\"2026-04-10 12:52:01.493255\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-25\"\n  \"ts\"=\"2026-04-10 12:52:01.493268\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-l8rh58\"\n  \"ts\"=\"2026-04-10 12:52:01.493296\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\"\n  \"ts\"=\"2026-04-10 12:52:01.646619\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:52:01.653898\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-44\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:52:01.685532\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-25\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 12:52:01.724961\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"pooledNonMsiScopedAssignments-igpwm4vimfr3y\"\n  \"ts\"=\"2026-04-10 12:52:01.776009\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\" \"deployment\"=\"msi-deployment-108\"\n  \"ts\"=\"2026-04-10 12:52:01.783539\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-44\" \"deployment\"=\"msi-deployment-44\"\n  \"ts\"=\"2026-04-10 12:52:01.867495\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"customer-infra-basic-hcp-cluster3-zvf5lf\"\n  \"ts\"=\"2026-04-10 12:52:02.032517\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-04-10 12:52:02.036082\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-25\" \"deployment\"=\"msi-deployment-25\"\n  \"ts\"=\"2026-04-10 12:52:02.328126\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-pg7qtn\"\n  \"ts\"=\"2026-04-10 12:52:03.086364\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-04-10 12:52:03.725551\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-l8rh58\" \"deployment\"=\"customer-infra-basic-hcp-cluster-m8xtvh\"\n  \"ts\"=\"2026-04-10 12:52:04.092347\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 12:52:04.092476\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 12:52:04.092575\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-l8rh58\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"f4031231f24bb154d12dede29e98dc4cf64d0c058bfccaade8532ad131dbd1e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","failed_at":"e2e","signature_id":"8b1222a8560e2917674611da65a4169d2346b2401f470a0385b6c34c7da27c8a","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:118]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e3a0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-7wck67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:118]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e3a0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-7wck67/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042524044317691904","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-10T08:44:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042523562169864192","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T08:42:20Z"},"failed_at":["e2e","unknown"],"failed_test_count":2,"occurrences":[{"row_id":"034b6b69e28927085d3f051bb1013041f87feaccae27e0b53903cb3202d4b5f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042523562169864192","occurred_at":"2026-04-10T08:42:20Z","failed_at":"e2e","signature_id":"977c5a79955b703d5d52fa873359cbadc299b6ed68168fe22a6de3d1a85024e4","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"idRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: 201: must be less than or equal to 200\\\",\\n    \\\"target\\\": \\\"properties.replicas\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: creating a nodepool @ 04/10/26 10:00:15.885\n    STEP: attempting to update nodepool version to higher than cluster version @ 04/10/26 10:30:37.398\n  \"ts\"=\"2026-04-10 10:30:38.320547\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zvckx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: attempting to update immutable platform profile fields @ 04/10/26 10:30:38.32\n  \"ts\"=\"2026-04-10 10:30:38.793824\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zvckx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"4 errors occurred\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  \"ts\"=\"2026-04-10 10:30:38.933364\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 10:30:38.933477\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-8w95lb\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\"\n  \"ts\"=\"2026-04-10 10:30:39.052915\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-8w95lb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 10:30:40.502197\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:30:40.502303\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:30:40.502363\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 10:30:40.503883\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-27\"\n  \"ts\"=\"2026-04-10 10:30:40.503901\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\"\n  \"ts\"=\"2026-04-10 10:30:40.649649\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n  \"ts\"=\"2026-04-10 10:30:40.710044\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\" \"deployment\"=\"customer-infra-negative-tests-cluster-8w95lb\"\n  \"ts\"=\"2026-04-10 10:30:41.332013\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-27\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 10:30:41.859258\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-27\" \"deployment\"=\"msi-deployment-27\"\n  \"ts\"=\"2026-04-10 10:30:42.052837\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 10:30:42.052956\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 10:30:42.053071\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: 201: must be less than or equal to 200\\\",\\n    \\\"target\\\": \\\"properties.replicas\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: creating a nodepool @ 04/10/26 10:00:15.885\n    STEP: attempting to update nodepool version to higher than cluster version @ 04/10/26 10:30:37.398\n  \"ts\"=\"2026-04-10 10:30:38.320547\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zvckx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n    \\\"target\\\": \\\"properties.version.id\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n    STEP: attempting to update immutable platform profile fields @ 04/10/26 10:30:38.32\n  \"ts\"=\"2026-04-10 10:30:38.793824\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-zvckx9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"4 errors occurred\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InvalidRequestContent\\\",\\n        \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n        \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  \"ts\"=\"2026-04-10 10:30:38.933364\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 10:30:38.933477\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-8w95lb\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\"\n  \"ts\"=\"2026-04-10 10:30:39.052915\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-8w95lb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 10:30:40.502197\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:30:40.502303\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:30:40.502363\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 10:30:40.503883\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-27\"\n  \"ts\"=\"2026-04-10 10:30:40.503901\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\"\n  \"ts\"=\"2026-04-10 10:30:40.649649\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n  \"ts\"=\"2026-04-10 10:30:40.710044\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\" \"deployment\"=\"customer-infra-negative-tests-cluster-8w95lb\"\n  \"ts\"=\"2026-04-10 10:30:41.332013\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-27\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 10:30:41.859258\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-27\" \"deployment\"=\"msi-deployment-27\"\n  \"ts\"=\"2026-04-10 10:30:42.052837\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 10:30:42.052956\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 10:30:42.053071\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-zvckx9\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"9d018798ca4d50006c0f4792f186fa115b92a13cc0a84700af7b3c89b7b580ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042523562169864192","occurred_at":"2026-04-10T08:42:20Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T10:44:38Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T08:37:16Z"},"failed_at":["e2e","unknown"],"failed_test_count":5,"occurrences":[{"row_id":"1c3567259a11a07884f6f4e01a806e7056bb45fa89f06db238da4224b53d0e1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","failed_at":"e2e","signature_id":"1b3bdf86f9c812b2a7b8f5c8ab5b543a1b2a695fcf36e4c92a1990c7902ccee1","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"===================\n  Random Seed: 1775813881 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\nW0410 09:38:01.837888     894 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 09:38:01.916295     894 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 09:38:01.916585\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:894\"\n  \"ts\"=\"2026-04-10 09:38:02.009014\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 09:38:02.009\n  \"ts\"=\"2026-04-10 09:38:02.009151\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 09:38:05.69\n    STEP: creating customer resources @ 04/10/26 09:38:05.788\n    STEP: creating the HCP cluster with version 4.20.17 @ 04/10/26 09:39:06.67\n  \"ts\"=\"2026-04-10 09:39:06.671088\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-xk7xd6\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\n    STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 09:52:44.143\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 10:11:12.29\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 10:13:24.961\n    STEP: capturing node release images before upgrade @ 04/10/26 10:13:25.137\n    STEP: triggering nodepool upgrade to version 4.20.17 @ 04/10/26 10:13:25.341\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 10:14:12.28\n    STEP: verifying node pool GET still reflects the new version @ 04/10/26 10:36:18.546\n  \"ts\"=\"2026-04-10 10:36:18.945641\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 10:36:18.945925\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-xk7xd6-mlggdr\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\n  \"ts\"=\"2026-04-10 10:36:19.435256\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-xk7xd6-mlggdr\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 10:36:20.855485\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:36:20.855726\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:36:20.855961\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 10:36:20.858515\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\"\n  \"ts\"=\"2026-04-10 10:36:20.858534\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\n  \"ts\"=\"2026-04-10 10:36:21.566430\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 10:36:21.718120\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\" \"deployment\"=\"msi-deployment-11\"\n  \"ts\"=\"2026-04-10 10:36:21.767633\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\" \"deployment\"=\"pooledNonMsiScopedAssignments-wlvrodc3sbnq4\"\n  \"ts\"=\"2026-04-10 10:36:22.824528\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-xk7xd6-mlggdr\"\n  \"ts\"=\"2026-04-10 10:36:23.837140\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 10:36:23.837297\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 10:36:23.837475\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n===================\n  Random Seed: 1775813881 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\nW0410 09:38:01.837888     894 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 09:38:01.916295     894 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 09:38:01.916585\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:894\"\n  \"ts\"=\"2026-04-10 09:38:02.009014\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 09:38:02.009\n  \"ts\"=\"2026-04-10 09:38:02.009151\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 09:38:05.69\n    STEP: creating customer resources @ 04/10/26 09:38:05.788\n    STEP: creating the HCP cluster with version 4.20.17 @ 04/10/26 09:39:06.67\n  \"ts\"=\"2026-04-10 09:39:06.671088\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-xk7xd6\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\n    STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 09:52:44.143\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 10:11:12.29\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 10:13:24.961\n    STEP: capturing node release images before upgrade @ 04/10/26 10:13:25.137\n    STEP: triggering nodepool upgrade to version 4.20.17 @ 04/10/26 10:13:25.341\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 10:14:12.28\n    STEP: verifying node pool GET still reflects the new version @ 04/10/26 10:36:18.546\n  \"ts\"=\"2026-04-10 10:36:18.945641\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 10:36:18.945925\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-xk7xd6-mlggdr\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\n  \"ts\"=\"2026-04-10 10:36:19.435256\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-xk7xd6-mlggdr\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 10:36:20.855485\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:36:20.855726\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:36:20.855961\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 10:36:20.858515\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\"\n  \"ts\"=\"2026-04-10 10:36:20.858534\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\n  \"ts\"=\"2026-04-10 10:36:21.566430\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 10:36:21.718120\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-11\" \"deployment\"=\"msi-deployment-11\"\n  \"ts\"=\"2026-04-10 10:36:21.767633\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\" \"deployment\"=\"pooledNonMsiScopedAssignments-wlvrodc3sbnq4\"\n  \"ts\"=\"2026-04-10 10:36:22.824528\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-xk7xd6-mlggdr\"\n  \"ts\"=\"2026-04-10 10:36:23.837140\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 10:36:23.837297\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 10:36:23.837475\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-xk7xd6-t4x7hz\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"6ba4728e814c92a2180395fff967c57b56bb85f914e1296919fe95168945e86c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","failed_at":"e2e","signature_id":"c269ac0dc1e4722962bb2c16446c66884aff962620fc94e198f5797dd1547de2","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775813878 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:223\nW0410 09:37:58.491380     881 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 09:37:58.569259     881 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 09:37:58.569629\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:881\"\n  \"ts\"=\"2026-04-10 09:38:01.525392\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"3s\"\n    STEP: creating resource group @ 04/10/26 09:38:01.525\n  \"ts\"=\"2026-04-10 09:38:01.525521\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-b77f69-wwx7xr\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 09:38:06.084\nW0410 09:38:06.171994     881 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: creating customer resources @ 04/10/26 09:38:06.172\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 09:38:58.106\n  \"ts\"=\"2026-04-10 09:38:58.106324\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-b77f69\" \"resourceGroup\"=\"rg-np-version-upgrade-b77f69-wwx7xr\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 09:53:35.552\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 10:17:50.709\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 10:20:21.895\nW0410 10:20:22.019930     881 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: capturing node release images before upgrade @ 04/10/26 10:20:22.019\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 10:20:22.208\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 10:21:09.14\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775813878 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:223\nW0410 09:37:58.491380     881 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 09:37:58.569259     881 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 09:37:58.569629\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:881\"\n  \"ts\"=\"2026-04-10 09:38:01.525392\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"3s\"\n    STEP: creating resource group @ 04/10/26 09:38:01.525\n  \"ts\"=\"2026-04-10 09:38:01.525521\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-b77f69-wwx7xr\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 09:38:06.084\nW0410 09:38:06.171994     881 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: creating customer resources @ 04/10/26 09:38:06.172\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 09:38:58.106\n  \"ts\"=\"2026-04-10 09:38:58.106324\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-b77f69\" \"resourceGroup\"=\"rg-np-version-upgrade-b77f69-wwx7xr\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 09:53:35.552\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 10:17:50.709\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 10:20:21.895\nW0410 10:20:22.019930     881 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: capturing node release images before upgrade @ 04/10/26 10:20:22.019\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 10:20:22.208\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 10:21:09.14\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e11e465cabdfb0f78262369438969272582dd576dd9e7aadc00ebc5a0f830984","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","failed_at":"e2e","signature_id":"774c8b9425c53dec4fa564c04d6b0b55f586c357fb5487ed255e12937b8edcaf","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"els\"={\"kubernetes.io/arch\"=\"amd64\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/zone\"=\"0\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"topology.kubernetes.io/region\"=\"westeurope\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/nodePool\"=\"m6j3f3x0x1m5j0h-np-1\" \"kubernetes.io/hostname\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-9pslf\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"node.openshift.io/os_id\"=\"rhel\" \"beta.kubernetes.io/arch\"=\"amd64\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n  \"ts\"=\"2026-04-10 10:33:52.686824\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-n9pqg\" \"labels\"={\"kubernetes.io/hostname\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-n9pqg\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/managed\"=\"true\" \"key2\"=\"value2\" \"node-role.kubernetes.io/worker\"=\"\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/region\"=\"westeurope\" \"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/nodePool\"=\"m6j3f3x0x1m5j0h-np-1\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\" \"kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n  \"ts\"=\"2026-04-10 10:33:52.686885\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-rkwb9\" \"labels\"={\"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"kubernetes.io/hostname\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-rkwb9\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/nodePool\"=\"m6j3f3x0x1m5j0h-np-1\" \"key1\"=\"value1\" \"node-role.kubernetes.io/worker\"=\"\" \"node.openshift.io/os_id\"=\"rhel\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"topology.kubernetes.io/zone\"=\"1\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n    STEP: verifying original labels persist on 3 nodes @ 04/10/26 10:33:52.686\n    STEP: verifying original taints persist on 2 nodes @ 04/10/26 10:33:52.686\n  \"ts\"=\"2026-04-10 10:33:52.687161\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 10:33:52.687268\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-6phmrh\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\"\n  \"ts\"=\"2026-04-10 10:33:52.888703\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-6phmrh\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 10:33:54.820943\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:33:54.821065\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:33:54.821146\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 10:33:54.822707\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\"\n  \"ts\"=\"2026-04-10 10:33:54.822719\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\"\n  \"ts\"=\"2026-04-10 10:33:55.000905\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 10:33:55.525859\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n  \"ts\"=\"2026-04-10 10:33:55.533515\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"msi-deployment-4\"\n  \"ts\"=\"2026-04-10 10:33:55.669144\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-6phmrh\"\n  \"ts\"=\"2026-04-10 10:33:56.557963\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 10:33:56.558106\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 10:33:56.558240\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nels\"={\"kubernetes.io/arch\"=\"amd64\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/zone\"=\"0\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"topology.kubernetes.io/region\"=\"westeurope\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/nodePool\"=\"m6j3f3x0x1m5j0h-np-1\" \"kubernetes.io/hostname\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-9pslf\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"node.openshift.io/os_id\"=\"rhel\" \"beta.kubernetes.io/arch\"=\"amd64\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n  \"ts\"=\"2026-04-10 10:33:52.686824\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-n9pqg\" \"labels\"={\"kubernetes.io/hostname\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-n9pqg\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/managed\"=\"true\" \"key2\"=\"value2\" \"node-role.kubernetes.io/worker\"=\"\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/region\"=\"westeurope\" \"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/nodePool\"=\"m6j3f3x0x1m5j0h-np-1\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\" \"kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n  \"ts\"=\"2026-04-10 10:33:52.686885\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-rkwb9\" \"labels\"={\"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"kubernetes.io/hostname\"=\"m6j3f3x0x1m5j0h-np-1-xw2qn-rkwb9\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/nodePool\"=\"m6j3f3x0x1m5j0h-np-1\" \"key1\"=\"value1\" \"node-role.kubernetes.io/worker\"=\"\" \"node.openshift.io/os_id\"=\"rhel\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"westeurope\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"topology.kubernetes.io/zone\"=\"1\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n    STEP: verifying original labels persist on 3 nodes @ 04/10/26 10:33:52.686\n    STEP: verifying original taints persist on 2 nodes @ 04/10/26 10:33:52.686\n  \"ts\"=\"2026-04-10 10:33:52.687161\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 10:33:52.687268\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-6phmrh\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\"\n  \"ts\"=\"2026-04-10 10:33:52.888703\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-6phmrh\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 10:33:54.820943\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:33:54.821065\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:33:54.821146\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 10:33:54.822707\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\"\n  \"ts\"=\"2026-04-10 10:33:54.822719\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\"\n  \"ts\"=\"2026-04-10 10:33:55.000905\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 10:33:55.525859\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n  \"ts\"=\"2026-04-10 10:33:55.533515\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"msi-deployment-4\"\n  \"ts\"=\"2026-04-10 10:33:55.669144\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-6phmrh\"\n  \"ts\"=\"2026-04-10 10:33:56.557963\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 10:33:56.558106\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 10:33:56.558240\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-c56s4p\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e8b8779ccc6daff9f80da535b1f3ddae461f4dd674a4603c223b7d42c6dcd83d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","failed_at":"e2e","signature_id":"eb113763ca9eb39cbfbf3f6d9b2910a4d656864f89ff1075609a1e9aa9129dd1","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"nly 0 are assigned\" \"attempt\"=11 \"retryIn\"=\"1m0s\" \"elapsed\"=\"10m0s\"\n  \"ts\"=\"2026-04-10 09:51:38.983444\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=12 \"retryIn\"=\"1m0s\" \"elapsed\"=\"11m0s\"\n  \"ts\"=\"2026-04-10 09:52:38.985566\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=13 \"retryIn\"=\"1m0s\" \"elapsed\"=\"12m0s\"\n  \"ts\"=\"2026-04-10 09:53:38.989067\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=14 \"retryIn\"=\"1m0s\" \"elapsed\"=\"13m0s\"\n  \"ts\"=\"2026-04-10 09:54:38.991008\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=15 \"retryIn\"=\"1m0s\" \"elapsed\"=\"14m0s\"\n  \"ts\"=\"2026-04-10 09:55:38.993317\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=16 \"retryIn\"=\"1m0s\" \"elapsed\"=\"15m0s\"\n  \"ts\"=\"2026-04-10 09:56:39.001085\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=17 \"elapsed\"=\"16m0s\"\n    STEP: creating a resource group @ 04/10/26 09:56:39.001\n  \"ts\"=\"2026-04-10 09:56:39.001171\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\n    STEP: creating cluster parameters @ 04/10/26 09:56:43.2\n    STEP: creating customer resources @ 04/10/26 09:56:43.2\n    STEP: creating the HCP cluster @ 04/10/26 09:57:45.497\n  \"ts\"=\"2026-04-10 09:57:45.497279\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\n    STEP: creating a storage account for shoebox logs @ 04/10/26 10:09:39.477\n  \"ts\"=\"2026-04-10 10:10:00.273925\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxm5mvvd\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-hfmst4/providers/Microsoft.Storage/storageAccounts/shoeboxm5mvvd\"\n    STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 10:10:00.273\n  \"ts\"=\"2026-04-10 10:10:02.370642\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n    STEP: waiting for log containers to appear in the storage account @ 04/10/26 10:10:02.37\n  \"ts\"=\"2026-04-10 10:38:02.717727\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxm5mvvd\" \"timestamp\"=\"2026-04-10T10:38:02Z\"\n  \"ts\"=\"2026-04-10 10:38:02.717995\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 10:38:02.718099\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-cvrd95\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\n  \"ts\"=\"2026-04-10 10:38:03.225584\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-cvrd95\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 10:38:05.408173\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:38:05.408414\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:38:05.408565\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 10:38:05.411248\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\"\n  \"ts\"=\"2026-04-10 10:38:05.411345\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\n  \"ts\"=\"2026-04-10 10:38:05.728184\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-10 10:38:05.767626\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 10:38:05.884292\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-cvrd95\"\n  \"ts\"=\"2026-04-10 10:38:05.908471\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\" \"deployment\"=\"msi-deployment-78\"\n  \"ts\"=\"2026-04-10 10:38:06.218123\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 10:38:06.218519\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 10:38:06.218829\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nnly 0 are assigned\" \"attempt\"=11 \"retryIn\"=\"1m0s\" \"elapsed\"=\"10m0s\"\n  \"ts\"=\"2026-04-10 09:51:38.983444\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=12 \"retryIn\"=\"1m0s\" \"elapsed\"=\"11m0s\"\n  \"ts\"=\"2026-04-10 09:52:38.985566\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=13 \"retryIn\"=\"1m0s\" \"elapsed\"=\"12m0s\"\n  \"ts\"=\"2026-04-10 09:53:38.989067\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=14 \"retryIn\"=\"1m0s\" \"elapsed\"=\"13m0s\"\n  \"ts\"=\"2026-04-10 09:54:38.991008\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=15 \"retryIn\"=\"1m0s\" \"elapsed\"=\"14m0s\"\n  \"ts\"=\"2026-04-10 09:55:38.993317\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=16 \"retryIn\"=\"1m0s\" \"elapsed\"=\"15m0s\"\n  \"ts\"=\"2026-04-10 09:56:39.001085\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=17 \"elapsed\"=\"16m0s\"\n    STEP: creating a resource group @ 04/10/26 09:56:39.001\n  \"ts\"=\"2026-04-10 09:56:39.001171\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\n    STEP: creating cluster parameters @ 04/10/26 09:56:43.2\n    STEP: creating customer resources @ 04/10/26 09:56:43.2\n    STEP: creating the HCP cluster @ 04/10/26 09:57:45.497\n  \"ts\"=\"2026-04-10 09:57:45.497279\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\n    STEP: creating a storage account for shoebox logs @ 04/10/26 10:09:39.477\n  \"ts\"=\"2026-04-10 10:10:00.273925\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxm5mvvd\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-hfmst4/providers/Microsoft.Storage/storageAccounts/shoeboxm5mvvd\"\n    STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 10:10:00.273\n  \"ts\"=\"2026-04-10 10:10:02.370642\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n    STEP: waiting for log containers to appear in the storage account @ 04/10/26 10:10:02.37\n  \"ts\"=\"2026-04-10 10:38:02.717727\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxm5mvvd\" \"timestamp\"=\"2026-04-10T10:38:02Z\"\n  \"ts\"=\"2026-04-10 10:38:02.717995\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 10:38:02.718099\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-cvrd95\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\n  \"ts\"=\"2026-04-10 10:38:03.225584\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-cvrd95\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 10:38:05.408173\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:38:05.408414\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 10:38:05.408565\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 10:38:05.411248\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\"\n  \"ts\"=\"2026-04-10 10:38:05.411345\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\n  \"ts\"=\"2026-04-10 10:38:05.728184\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-10 10:38:05.767626\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 10:38:05.884292\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-cvrd95\"\n  \"ts\"=\"2026-04-10 10:38:05.908471\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-78\" \"deployment\"=\"msi-deployment-78\"\n  \"ts\"=\"2026-04-10 10:38:06.218123\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 10:38:06.218519\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 10:38:06.218829\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-hfmst4\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"fd9f09783de35f0dc316d91d4411ddd35162a815c6c526649da7b17ab891c6ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T10:42:24Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-10T02:06:29Z"},"failed_at":["e2e","unknown"],"failed_test_count":21,"occurrences":[{"row_id":"2bedd3ab5be0e649ede31a33d9ed54e62f899ffe4568ac21ecdbb59e8eb5299c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"85b95c9895e4e0f7c2fd44a5c6864b35807d9f8a75a3b9bb1666804bfe472133","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"\"ts\"=\"2026-04-10 03:43:20.625744\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=93 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h32m0s\"\n  \"ts\"=\"2026-04-10 03:44:20.628519\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=94 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h33m0s\"\n  \"ts\"=\"2026-04-10 03:45:20.631193\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=95 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h34m0s\"\n  \"ts\"=\"2026-04-10 03:46:20.633847\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=96 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h35m0s\"\n  \"ts\"=\"2026-04-10 03:47:20.636270\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=97 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h36m0s\"\n  \"ts\"=\"2026-04-10 03:48:20.642102\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=98 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h37m0s\"\n  \"ts\"=\"2026-04-10 03:49:20.645334\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=99 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h38m0s\"\n  \"ts\"=\"2026-04-10 03:50:20.654693\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=100 \"elapsed\"=\"1h39m0s\"\n    STEP: creating a resource group @ 04/10/26 03:50:20.654\n  \"ts\"=\"2026-04-10 03:50:20.654798\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\n    STEP: creating cluster parameters @ 04/10/26 03:50:24.858\n    STEP: creating customer resources @ 04/10/26 03:50:24.858\n    STEP: creating the HCP cluster @ 04/10/26 03:51:33.679\n  \"ts\"=\"2026-04-10 03:51:33.679728\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"patch-tags-cluster\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\n    STEP: getting credentials @ 04/10/26 04:07:39.558\n    STEP: ensuring the cluster is viable @ 04/10/26 04:09:44.551\n    STEP: sending a PATCH request to set a tag @ 04/10/26 04:09:45.833\n    STEP: verifying the tag is present in the update response body @ 04/10/26 04:10:42.758\n    STEP: verifying the tag is present on the cluster @ 04/10/26 04:10:42.758\n  \"ts\"=\"2026-04-10 04:10:43.153297\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 04:10:43.153395\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-tags-cluster-5bk5cn\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\n  \"ts\"=\"2026-04-10 04:10:43.442391\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-tags-cluster-5bk5cn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 04:10:45.963000\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:10:45.963106\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:10:45.963168\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 04:10:45.964991\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-64\"\n  \"ts\"=\"2026-04-10 04:10:45.965015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\n  \"ts\"=\"2026-04-10 04:10:46.141081\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-64\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 04:10:46.264710\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-64\" \"deployment\"=\"msi-deployment-64\"\n  \"ts\"=\"2026-04-10 04:10:46.277682\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-xkxpmj\" \"deployment\"=\"pooledNonMsiScopedAssignments-fklo75htpxzxc\"\n  \"ts\"=\"2026-04-10 04:10:46.545512\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-xkxpmj\" \"deployment\"=\"customer-infra-patch-tags-cluster-5bk5cn\"\n  \"ts\"=\"2026-04-10 04:10:46.811374\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 04:10:46.811511\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 04:10:46.811625\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\"ts\"=\"2026-04-10 03:43:20.625744\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=93 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h32m0s\"\n  \"ts\"=\"2026-04-10 03:44:20.628519\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=94 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h33m0s\"\n  \"ts\"=\"2026-04-10 03:45:20.631193\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=95 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h34m0s\"\n  \"ts\"=\"2026-04-10 03:46:20.633847\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=96 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h35m0s\"\n  \"ts\"=\"2026-04-10 03:47:20.636270\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=97 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h36m0s\"\n  \"ts\"=\"2026-04-10 03:48:20.642102\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=98 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h37m0s\"\n  \"ts\"=\"2026-04-10 03:49:20.645334\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=99 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h38m0s\"\n  \"ts\"=\"2026-04-10 03:50:20.654693\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=100 \"elapsed\"=\"1h39m0s\"\n    STEP: creating a resource group @ 04/10/26 03:50:20.654\n  \"ts\"=\"2026-04-10 03:50:20.654798\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\n    STEP: creating cluster parameters @ 04/10/26 03:50:24.858\n    STEP: creating customer resources @ 04/10/26 03:50:24.858\n    STEP: creating the HCP cluster @ 04/10/26 03:51:33.679\n  \"ts\"=\"2026-04-10 03:51:33.679728\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"patch-tags-cluster\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\n    STEP: getting credentials @ 04/10/26 04:07:39.558\n    STEP: ensuring the cluster is viable @ 04/10/26 04:09:44.551\n    STEP: sending a PATCH request to set a tag @ 04/10/26 04:09:45.833\n    STEP: verifying the tag is present in the update response body @ 04/10/26 04:10:42.758\n    STEP: verifying the tag is present on the cluster @ 04/10/26 04:10:42.758\n  \"ts\"=\"2026-04-10 04:10:43.153297\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 04:10:43.153395\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-tags-cluster-5bk5cn\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\n  \"ts\"=\"2026-04-10 04:10:43.442391\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-tags-cluster-5bk5cn\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 04:10:45.963000\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:10:45.963106\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:10:45.963168\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 04:10:45.964991\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-64\"\n  \"ts\"=\"2026-04-10 04:10:45.965015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\n  \"ts\"=\"2026-04-10 04:10:46.141081\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-64\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 04:10:46.264710\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-64\" \"deployment\"=\"msi-deployment-64\"\n  \"ts\"=\"2026-04-10 04:10:46.277682\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-xkxpmj\" \"deployment\"=\"pooledNonMsiScopedAssignments-fklo75htpxzxc\"\n  \"ts\"=\"2026-04-10 04:10:46.545512\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-xkxpmj\" \"deployment\"=\"customer-infra-patch-tags-cluster-5bk5cn\"\n  \"ts\"=\"2026-04-10 04:10:46.811374\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 04:10:46.811511\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 04:10:46.811625\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-tags-xkxpmj\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"30f7a0ecee3d6d69a66cbdaec5842ba11990c56fa2fb77fbf8baf5a750a2766b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"3d519ffa44179193aedca896e5792ca8b7df0696ac0a7f979d1d282a5702f1c9","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00087a680\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9w9nvc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00087a680\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9w9nvc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"467d017764dfe592c334a4513b508d0e08379744956baffde3d62b66c2a54e10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"f2fc8aa99648d2d62ceefe9d3e42b8eb17da274af35f704f60c14515a7ecceb4","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a8a220\u003e: \n    failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-fhjd2t\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-fhjd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a8a220\u003e: \n    failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-fhjd2t\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-fhjd2t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"52ea76c577d181c853bf176746b25d36ed254b21818c3dfa4b6aa1b94d56931c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"a09d0db5dedab2fb189aca9a3aad2190a32644a96b9529a61aa4af99f28d3beb","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"ee identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=37 \"retryIn\"=\"1m0s\" \"elapsed\"=\"36m0s\"\n  \"ts\"=\"2026-04-10 03:44:28.728317\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=38 \"retryIn\"=\"1m0s\" \"elapsed\"=\"37m0s\"\n  \"ts\"=\"2026-04-10 03:45:28.731638\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=39 \"retryIn\"=\"1m0s\" \"elapsed\"=\"38m0s\"\n  \"ts\"=\"2026-04-10 03:46:28.734214\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=40 \"retryIn\"=\"1m0s\" \"elapsed\"=\"39m0s\"\n  \"ts\"=\"2026-04-10 03:47:28.737026\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=41 \"retryIn\"=\"1m0s\" \"elapsed\"=\"40m0s\"\n  \"ts\"=\"2026-04-10 03:48:28.739145\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=42 \"retryIn\"=\"1m0s\" \"elapsed\"=\"41m0s\"\n  \"ts\"=\"2026-04-10 03:49:28.741381\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=43 \"retryIn\"=\"1m0s\" \"elapsed\"=\"42m0s\"\n  \"ts\"=\"2026-04-10 03:50:28.743535\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=44 \"retryIn\"=\"1m0s\" \"elapsed\"=\"43m0s\"\n  \"ts\"=\"2026-04-10 03:51:28.746782\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=45 \"retryIn\"=\"1m0s\" \"elapsed\"=\"44m0s\"\n  \"ts\"=\"2026-04-10 03:52:28.749783\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=46 \"retryIn\"=\"1m0s\" \"elapsed\"=\"45m0s\"\n  \"ts\"=\"2026-04-10 03:53:28.753253\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=47 \"retryIn\"=\"1m0s\" \"elapsed\"=\"46m0s\"\n  \"ts\"=\"2026-04-10 03:54:28.755799\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=48 \"retryIn\"=\"1m0s\" \"elapsed\"=\"47m0s\"\n  \"ts\"=\"2026-04-10 03:55:28.759054\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=49 \"retryIn\"=\"1m0s\" \"elapsed\"=\"48m0s\"\n  \"ts\"=\"2026-04-10 03:56:28.760633\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=50 \"retryIn\"=\"1m0s\" \"elapsed\"=\"49m0s\"\n  \"ts\"=\"2026-04-10 03:57:28.763733\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=51 \"retryIn\"=\"1m0s\" \"elapsed\"=\"50m0s\"\n  \"ts\"=\"2026-04-10 03:58:28.766632\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=52 \"retryIn\"=\"1m0s\" \"elapsed\"=\"51m0s\"\n  \"ts\"=\"2026-04-10 03:59:28.769759\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=53 \"retryIn\"=\"1m0s\" \"elapsed\"=\"52m0s\"\n  \"ts\"=\"2026-04-10 04:00:28.782533\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=54 \"elapsed\"=\"53m0s\"\n    STEP: creating resource group for the HCP cluster @ 04/10/26 04:00:28.782\n  \"ts\"=\"2026-04-10 04:00:28.782639\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"ea-list-tsxbpj\"\n    STEP: creating cluster parameters @ 04/10/26 04:00:32.482\n    STEP: creating customer resources @ 04/10/26 04:00:32.482\n    STEP: creating HCP cluster @ 04/10/26 04:01:54.359\n  \"ts\"=\"2026-04-10 04:01:54.359988\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ea-listkw785z\" \"resourceGroup\"=\"ea-list-tsxbpj\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nee identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=37 \"retryIn\"=\"1m0s\" \"elapsed\"=\"36m0s\"\n  \"ts\"=\"2026-04-10 03:44:28.728317\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=38 \"retryIn\"=\"1m0s\" \"elapsed\"=\"37m0s\"\n  \"ts\"=\"2026-04-10 03:45:28.731638\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=39 \"retryIn\"=\"1m0s\" \"elapsed\"=\"38m0s\"\n  \"ts\"=\"2026-04-10 03:46:28.734214\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=40 \"retryIn\"=\"1m0s\" \"elapsed\"=\"39m0s\"\n  \"ts\"=\"2026-04-10 03:47:28.737026\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=41 \"retryIn\"=\"1m0s\" \"elapsed\"=\"40m0s\"\n  \"ts\"=\"2026-04-10 03:48:28.739145\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=42 \"retryIn\"=\"1m0s\" \"elapsed\"=\"41m0s\"\n  \"ts\"=\"2026-04-10 03:49:28.741381\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=43 \"retryIn\"=\"1m0s\" \"elapsed\"=\"42m0s\"\n  \"ts\"=\"2026-04-10 03:50:28.743535\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=44 \"retryIn\"=\"1m0s\" \"elapsed\"=\"43m0s\"\n  \"ts\"=\"2026-04-10 03:51:28.746782\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=45 \"retryIn\"=\"1m0s\" \"elapsed\"=\"44m0s\"\n  \"ts\"=\"2026-04-10 03:52:28.749783\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=46 \"retryIn\"=\"1m0s\" \"elapsed\"=\"45m0s\"\n  \"ts\"=\"2026-04-10 03:53:28.753253\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=47 \"retryIn\"=\"1m0s\" \"elapsed\"=\"46m0s\"\n  \"ts\"=\"2026-04-10 03:54:28.755799\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=48 \"retryIn\"=\"1m0s\" \"elapsed\"=\"47m0s\"\n  \"ts\"=\"2026-04-10 03:55:28.759054\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=49 \"retryIn\"=\"1m0s\" \"elapsed\"=\"48m0s\"\n  \"ts\"=\"2026-04-10 03:56:28.760633\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=50 \"retryIn\"=\"1m0s\" \"elapsed\"=\"49m0s\"\n  \"ts\"=\"2026-04-10 03:57:28.763733\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=51 \"retryIn\"=\"1m0s\" \"elapsed\"=\"50m0s\"\n  \"ts\"=\"2026-04-10 03:58:28.766632\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=52 \"retryIn\"=\"1m0s\" \"elapsed\"=\"51m0s\"\n  \"ts\"=\"2026-04-10 03:59:28.769759\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=53 \"retryIn\"=\"1m0s\" \"elapsed\"=\"52m0s\"\n  \"ts\"=\"2026-04-10 04:00:28.782533\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=54 \"elapsed\"=\"53m0s\"\n    STEP: creating resource group for the HCP cluster @ 04/10/26 04:00:28.782\n  \"ts\"=\"2026-04-10 04:00:28.782639\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"ea-list-tsxbpj\"\n    STEP: creating cluster parameters @ 04/10/26 04:00:32.482\n    STEP: creating customer resources @ 04/10/26 04:00:32.482\n    STEP: creating HCP cluster @ 04/10/26 04:01:54.359\n  \"ts\"=\"2026-04-10 04:01:54.359988\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ea-listkw785z\" \"resourceGroup\"=\"ea-list-tsxbpj\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"539c354c7b23ad1eb3ea7b806f2b4b4a155b100073a8e510d0dec29460d461f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"c79c0556df8ad3acfeb27115565aa651ad5885ac90240887372b6c08276ea8a9","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775791117 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:223\nW0410 03:18:37.270340     875 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 03:18:37.350291     875 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 03:18:37.350682\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:875\"\n  \"ts\"=\"2026-04-10 03:18:37.359639\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 03:18:37.359\n  \"ts\"=\"2026-04-10 03:18:37.359757\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-qs7b4k-lv9m85\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 03:18:41.298\nW0410 03:18:41.389740     875 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: creating customer resources @ 04/10/26 03:18:41.389\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 03:19:52.126\n  \"ts\"=\"2026-04-10 03:19:52.126471\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-qs7b4k\" \"resourceGroup\"=\"rg-np-version-upgrade-qs7b4k-lv9m85\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 03:36:40.072\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 03:46:19.01\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 03:48:21.962\nW0410 03:48:22.094001     875 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: capturing node release images before upgrade @ 04/10/26 03:48:22.094\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 03:48:22.512\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 03:49:07.595\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775791117 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:223\nW0410 03:18:37.270340     875 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 03:18:37.350291     875 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n  \"ts\"=\"2026-04-10 03:18:37.350682\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:875\"\n  \"ts\"=\"2026-04-10 03:18:37.359639\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating resource group @ 04/10/26 03:18:37.359\n  \"ts\"=\"2026-04-10 03:18:37.359757\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-qs7b4k-lv9m85\"\n    STEP: creating cluster parameters at control plane version @ 04/10/26 03:18:41.298\nW0410 03:18:41.389740     875 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: creating customer resources @ 04/10/26 03:18:41.389\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 03:19:52.126\n  \"ts\"=\"2026-04-10 03:19:52.126471\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-qs7b4k\" \"resourceGroup\"=\"rg-np-version-upgrade-qs7b4k-lv9m85\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 03:36:40.072\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 03:46:19.01\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 03:48:21.962\nW0410 03:48:22.094001     875 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n    STEP: capturing node release images before upgrade @ 04/10/26 03:48:22.094\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 03:48:22.512\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 03:49:07.595\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"57913857909a0f7887e14b74f7d628eee9b136330ba33ce2839d5521349863c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"cb1103453558f656c64bcf405755d3e4078107305cbaf3d067fbc78e283e95ac","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"\"ts\"=\"2026-04-10 03:37:20.814830\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=31 \"retryIn\"=\"1m0s\" \"elapsed\"=\"30m0s\"\n  \"ts\"=\"2026-04-10 03:38:20.817653\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=32 \"retryIn\"=\"1m0s\" \"elapsed\"=\"31m0s\"\n  \"ts\"=\"2026-04-10 03:39:20.820524\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=33 \"retryIn\"=\"1m0s\" \"elapsed\"=\"32m0s\"\n  \"ts\"=\"2026-04-10 03:40:20.822516\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=34 \"retryIn\"=\"1m0s\" \"elapsed\"=\"33m0s\"\n  \"ts\"=\"2026-04-10 03:41:20.825054\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=35 \"retryIn\"=\"1m0s\" \"elapsed\"=\"34m0s\"\n  \"ts\"=\"2026-04-10 03:42:20.827784\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=36 \"retryIn\"=\"1m0s\" \"elapsed\"=\"35m0s\"\n  \"ts\"=\"2026-04-10 03:43:20.831953\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=37 \"retryIn\"=\"1m0s\" \"elapsed\"=\"36m0s\"\n  \"ts\"=\"2026-04-10 03:44:20.834246\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=38 \"retryIn\"=\"1m0s\" \"elapsed\"=\"37m0s\"\n  \"ts\"=\"2026-04-10 03:45:20.838247\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=39 \"retryIn\"=\"1m0s\" \"elapsed\"=\"38m0s\"\n  \"ts\"=\"2026-04-10 03:46:20.840670\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=40 \"retryIn\"=\"1m0s\" \"elapsed\"=\"39m0s\"\n  \"ts\"=\"2026-04-10 03:47:20.843513\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=41 \"retryIn\"=\"1m0s\" \"elapsed\"=\"40m0s\"\n  \"ts\"=\"2026-04-10 03:48:20.845948\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=42 \"retryIn\"=\"1m0s\" \"elapsed\"=\"41m0s\"\n  \"ts\"=\"2026-04-10 03:49:20.848662\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=43 \"retryIn\"=\"1m0s\" \"elapsed\"=\"42m0s\"\n  \"ts\"=\"2026-04-10 03:50:20.851282\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=44 \"retryIn\"=\"1m0s\" \"elapsed\"=\"43m0s\"\n  \"ts\"=\"2026-04-10 03:51:20.854312\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=45 \"retryIn\"=\"1m0s\" \"elapsed\"=\"44m0s\"\n  \"ts\"=\"2026-04-10 03:52:20.857003\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=46 \"retryIn\"=\"1m0s\" \"elapsed\"=\"45m0s\"\n  \"ts\"=\"2026-04-10 03:53:20.889041\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=47 \"elapsed\"=\"46m0s\"\n    STEP: creating resource group @ 04/10/26 03:53:20.889\n  \"ts\"=\"2026-04-10 03:53:20.889167\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-cp-ystream-upgrade-4-21-q6jzn7\"\n    STEP: creating cluster parameters at install (previous minor) version @ 04/10/26 03:53:24.652\n    STEP: creating customer resources @ 04/10/26 03:53:24.652\n    STEP: creating the HCP cluster with install version 4.20.17 (previous minor 4.20.0) @ 04/10/26 03:54:45.281\n  \"ts\"=\"2026-04-10 03:54:45.282075\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cp-ystream-upgrade-4-21-9vfdjd\" \"resourceGroup\"=\"rg-cp-ystream-upgrade-4-21-q6jzn7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\"ts\"=\"2026-04-10 03:37:20.814830\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=31 \"retryIn\"=\"1m0s\" \"elapsed\"=\"30m0s\"\n  \"ts\"=\"2026-04-10 03:38:20.817653\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=32 \"retryIn\"=\"1m0s\" \"elapsed\"=\"31m0s\"\n  \"ts\"=\"2026-04-10 03:39:20.820524\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=33 \"retryIn\"=\"1m0s\" \"elapsed\"=\"32m0s\"\n  \"ts\"=\"2026-04-10 03:40:20.822516\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=34 \"retryIn\"=\"1m0s\" \"elapsed\"=\"33m0s\"\n  \"ts\"=\"2026-04-10 03:41:20.825054\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=35 \"retryIn\"=\"1m0s\" \"elapsed\"=\"34m0s\"\n  \"ts\"=\"2026-04-10 03:42:20.827784\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=36 \"retryIn\"=\"1m0s\" \"elapsed\"=\"35m0s\"\n  \"ts\"=\"2026-04-10 03:43:20.831953\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=37 \"retryIn\"=\"1m0s\" \"elapsed\"=\"36m0s\"\n  \"ts\"=\"2026-04-10 03:44:20.834246\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=38 \"retryIn\"=\"1m0s\" \"elapsed\"=\"37m0s\"\n  \"ts\"=\"2026-04-10 03:45:20.838247\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=39 \"retryIn\"=\"1m0s\" \"elapsed\"=\"38m0s\"\n  \"ts\"=\"2026-04-10 03:46:20.840670\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=40 \"retryIn\"=\"1m0s\" \"elapsed\"=\"39m0s\"\n  \"ts\"=\"2026-04-10 03:47:20.843513\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=41 \"retryIn\"=\"1m0s\" \"elapsed\"=\"40m0s\"\n  \"ts\"=\"2026-04-10 03:48:20.845948\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=42 \"retryIn\"=\"1m0s\" \"elapsed\"=\"41m0s\"\n  \"ts\"=\"2026-04-10 03:49:20.848662\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=43 \"retryIn\"=\"1m0s\" \"elapsed\"=\"42m0s\"\n  \"ts\"=\"2026-04-10 03:50:20.851282\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=44 \"retryIn\"=\"1m0s\" \"elapsed\"=\"43m0s\"\n  \"ts\"=\"2026-04-10 03:51:20.854312\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=45 \"retryIn\"=\"1m0s\" \"elapsed\"=\"44m0s\"\n  \"ts\"=\"2026-04-10 03:52:20.857003\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=46 \"retryIn\"=\"1m0s\" \"elapsed\"=\"45m0s\"\n  \"ts\"=\"2026-04-10 03:53:20.889041\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=47 \"elapsed\"=\"46m0s\"\n    STEP: creating resource group @ 04/10/26 03:53:20.889\n  \"ts\"=\"2026-04-10 03:53:20.889167\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-cp-ystream-upgrade-4-21-q6jzn7\"\n    STEP: creating cluster parameters at install (previous minor) version @ 04/10/26 03:53:24.652\n    STEP: creating customer resources @ 04/10/26 03:53:24.652\n    STEP: creating the HCP cluster with install version 4.20.17 (previous minor 4.20.0) @ 04/10/26 03:54:45.281\n  \"ts\"=\"2026-04-10 03:54:45.282075\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cp-ystream-upgrade-4-21-9vfdjd\" \"resourceGroup\"=\"rg-cp-ystream-upgrade-4-21-q6jzn7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"6022f7373afaeb21608c318651702bd386fa6b4cf558de39f2c5a2ba5bca7b9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"456956c00d2cd03fe0e688f32f7d08a0fab5648b265b6120efc3254d3963f16b","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775791077 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:67\n  \"ts\"=\"2026-04-10 03:17:57.722076\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Nodepool-Ephemeral-OS-Disk-should-create-a-nodepool-with-ephemeral-OS-disk-when-autoRepair-is-enabled|pid:850\"\n  \"ts\"=\"2026-04-10 03:17:57.728052\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 03:17:57.728\n  \"ts\"=\"2026-04-10 03:17:57.728100\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\n    STEP: creating cluster parameters @ 04/10/26 03:18:01.893\n    STEP: creating customer resources (infrastructure and managed identities) @ 04/10/26 03:18:01.893\n    STEP: creating the HCP cluster @ 04/10/26 03:19:19.787\n  \"ts\"=\"2026-04-10 03:19:19.788115\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ephemeral-disk\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112 @ 04/10/26 04:04:19.788\n  \"ts\"=\"2026-04-10 04:04:19.789009\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 04:04:19.789143\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ephemeral-disk-6b7cpt\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\n  \"ts\"=\"2026-04-10 04:04:20.070933\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ephemeral-disk-6b7cpt\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 04:04:22.168165\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a54a7b95d75fb7a5c86e976d7b6a232359fe80ddc62a98537cfa2cfbd9f24bac.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:04:22.168254\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a54a7b95d75fb7a5c86e976d7b6a232359fe80ddc62a98537cfa2cfbd9f24bac.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:04:22.168310\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 04:04:22.169635\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\"\n  \"ts\"=\"2026-04-10 04:04:22.169648\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\n  \"ts\"=\"2026-04-10 04:04:22.464446\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\" \"deployment\"=\"pooledNonMsiScopedAssignments-6xts4ycasve7q\"\n  \"ts\"=\"2026-04-10 04:04:22.680743\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 04:04:22.744099\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\" \"deployment\"=\"customer-infra-ephemeral-disk-6b7cpt\"\n  \"ts\"=\"2026-04-10 04:04:22.827305\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\" \"deployment\"=\"msi-deployment-53\"\n  \"ts\"=\"2026-04-10 04:04:23.041143\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 04:04:23.041245\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 04:04:23.041354\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775791077 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:67\n  \"ts\"=\"2026-04-10 03:17:57.722076\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Nodepool-Ephemeral-OS-Disk-should-create-a-nodepool-with-ephemeral-OS-disk-when-autoRepair-is-enabled|pid:850\"\n  \"ts\"=\"2026-04-10 03:17:57.728052\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 03:17:57.728\n  \"ts\"=\"2026-04-10 03:17:57.728100\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\n    STEP: creating cluster parameters @ 04/10/26 03:18:01.893\n    STEP: creating customer resources (infrastructure and managed identities) @ 04/10/26 03:18:01.893\n    STEP: creating the HCP cluster @ 04/10/26 03:19:19.787\n  \"ts\"=\"2026-04-10 03:19:19.788115\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"ephemeral-disk\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112 @ 04/10/26 04:04:19.788\n  \"ts\"=\"2026-04-10 04:04:19.789009\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 04:04:19.789143\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-ephemeral-disk-6b7cpt\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\n  \"ts\"=\"2026-04-10 04:04:20.070933\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-ephemeral-disk-6b7cpt\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 04:04:22.168165\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a54a7b95d75fb7a5c86e976d7b6a232359fe80ddc62a98537cfa2cfbd9f24bac.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:04:22.168254\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a54a7b95d75fb7a5c86e976d7b6a232359fe80ddc62a98537cfa2cfbd9f24bac.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:04:22.168310\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 04:04:22.169635\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\"\n  \"ts\"=\"2026-04-10 04:04:22.169648\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\n  \"ts\"=\"2026-04-10 04:04:22.464446\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\" \"deployment\"=\"pooledNonMsiScopedAssignments-6xts4ycasve7q\"\n  \"ts\"=\"2026-04-10 04:04:22.680743\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 04:04:22.744099\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\" \"deployment\"=\"customer-infra-ephemeral-disk-6b7cpt\"\n  \"ts\"=\"2026-04-10 04:04:22.827305\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-53\" \"deployment\"=\"msi-deployment-53\"\n  \"ts\"=\"2026-04-10 04:04:23.041143\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 04:04:23.041245\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 04:04:23.041354\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"ephemeral-osdisk-6n9k4m\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"6647fa9e595ed9029367ccc44c4c19a254722198b40aa276e06286f1c3a67d7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"ba350b37506f60e30bb4bb27da0434f337a2464ab0c305b5ebf8b0a627f56e6c","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009893e0\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-rk2pg2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-rk2pg2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009893e0\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-rk2pg2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-rk2pg2, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"827db73f56dcef45d940d6a791aed0ea547ab56bc1d275c96f06c8c8cdde376f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"5012567a501f3d37567b8d2827f0fd6d46f0b92140488b498409244711ec15f7","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f2c840\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qczcv4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-qczcv4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f2c840\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qczcv4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-qczcv4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}},{"row_id":"8ab7865e833a25a169952665a4aafed193a14cf0c7c11e0d1aebfafc29ee8621","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"711813ff0ef173ae4c74b1746c0d4883d500104597abcceb58cf5fd7e1fa6fb9","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"=\"1m0s\" \"elapsed\"=\"1h20m0s\"\n  \"ts\"=\"2026-04-10 03:32:20.590832\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=82 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h21m0s\"\n  \"ts\"=\"2026-04-10 03:33:20.593869\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=83 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h22m0s\"\n  \"ts\"=\"2026-04-10 03:34:20.597155\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=84 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h23m0s\"\n  \"ts\"=\"2026-04-10 03:35:20.599426\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=85 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h24m0s\"\n  \"ts\"=\"2026-04-10 03:36:20.602757\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=86 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h25m0s\"\n  \"ts\"=\"2026-04-10 03:37:20.604536\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=87 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h26m0s\"\n  \"ts\"=\"2026-04-10 03:38:20.606971\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=88 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h27m0s\"\n  \"ts\"=\"2026-04-10 03:39:20.609391\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=89 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h28m0s\"\n  \"ts\"=\"2026-04-10 03:40:20.612976\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=90 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h29m0s\"\n  \"ts\"=\"2026-04-10 03:41:20.615650\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=91 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h30m0s\"\n  \"ts\"=\"2026-04-10 03:42:20.618547\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=92 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h31m0s\"\n  \"ts\"=\"2026-04-10 03:43:20.621167\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=93 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h32m0s\"\n  \"ts\"=\"2026-04-10 03:44:20.624472\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=94 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h33m0s\"\n  \"ts\"=\"2026-04-10 03:45:20.626601\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=95 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h34m0s\"\n  \"ts\"=\"2026-04-10 03:46:20.629740\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=96 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h35m0s\"\n  \"ts\"=\"2026-04-10 03:47:20.631439\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=97 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h36m0s\"\n  \"ts\"=\"2026-04-10 03:48:20.640252\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=98 \"elapsed\"=\"1h37m0s\"\n    STEP: creating a resource group @ 04/10/26 03:48:20.64\n  \"ts\"=\"2026-04-10 03:48:20.640351\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"private-keyvault-5rpxss\"\n    STEP: creating cluster parameters @ 04/10/26 03:48:24.624\n    STEP: creating customer resources (infrastructure and managed identities) @ 04/10/26 03:48:24.624\n    STEP: creating the HCP cluster @ 04/10/26 03:52:10.448\n  \"ts\"=\"2026-04-10 03:52:10.449153\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation (v20251223preview)\" \"clusterName\"=\"private-kv-cluster\" \"resourceGroup\"=\"private-keyvault-5rpxss\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n=\"1m0s\" \"elapsed\"=\"1h20m0s\"\n  \"ts\"=\"2026-04-10 03:32:20.590832\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=82 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h21m0s\"\n  \"ts\"=\"2026-04-10 03:33:20.593869\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=83 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h22m0s\"\n  \"ts\"=\"2026-04-10 03:34:20.597155\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=84 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h23m0s\"\n  \"ts\"=\"2026-04-10 03:35:20.599426\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=85 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h24m0s\"\n  \"ts\"=\"2026-04-10 03:36:20.602757\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=86 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h25m0s\"\n  \"ts\"=\"2026-04-10 03:37:20.604536\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=87 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h26m0s\"\n  \"ts\"=\"2026-04-10 03:38:20.606971\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=88 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h27m0s\"\n  \"ts\"=\"2026-04-10 03:39:20.609391\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=89 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h28m0s\"\n  \"ts\"=\"2026-04-10 03:40:20.612976\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=90 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h29m0s\"\n  \"ts\"=\"2026-04-10 03:41:20.615650\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=91 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h30m0s\"\n  \"ts\"=\"2026-04-10 03:42:20.618547\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=92 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h31m0s\"\n  \"ts\"=\"2026-04-10 03:43:20.621167\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=93 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h32m0s\"\n  \"ts\"=\"2026-04-10 03:44:20.624472\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=94 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h33m0s\"\n  \"ts\"=\"2026-04-10 03:45:20.626601\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=95 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h34m0s\"\n  \"ts\"=\"2026-04-10 03:46:20.629740\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=96 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h35m0s\"\n  \"ts\"=\"2026-04-10 03:47:20.631439\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=97 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h36m0s\"\n  \"ts\"=\"2026-04-10 03:48:20.640252\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=98 \"elapsed\"=\"1h37m0s\"\n    STEP: creating a resource group @ 04/10/26 03:48:20.64\n  \"ts\"=\"2026-04-10 03:48:20.640351\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"private-keyvault-5rpxss\"\n    STEP: creating cluster parameters @ 04/10/26 03:48:24.624\n    STEP: creating customer resources (infrastructure and managed identities) @ 04/10/26 03:48:24.624\n    STEP: creating the HCP cluster @ 04/10/26 03:52:10.448\n  \"ts\"=\"2026-04-10 03:52:10.449153\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation (v20251223preview)\" \"clusterName\"=\"private-kv-cluster\" \"resourceGroup\"=\"private-keyvault-5rpxss\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"8d136ab7b9b1a1df12c5405507c2e00daee1f80565c53cdece338f36dbbf9174","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"a4e2d1c4ce37762825496b9afd8965783d6fd32cc144b275c851da1948e6b75e","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"\"=\"1m0s\" \"elapsed\"=\"1h23m0s\"\n  \"ts\"=\"2026-04-10 03:35:20.604098\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=85 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h24m0s\"\n  \"ts\"=\"2026-04-10 03:36:20.607920\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=86 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h25m0s\"\n  \"ts\"=\"2026-04-10 03:37:20.609946\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=87 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h26m0s\"\n  \"ts\"=\"2026-04-10 03:38:20.612631\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=88 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h27m0s\"\n  \"ts\"=\"2026-04-10 03:39:20.614440\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=89 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h28m0s\"\n  \"ts\"=\"2026-04-10 03:40:20.618754\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=90 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h29m0s\"\n  \"ts\"=\"2026-04-10 03:41:20.621295\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=91 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h30m0s\"\n  \"ts\"=\"2026-04-10 03:42:20.624572\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=92 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h31m0s\"\n  \"ts\"=\"2026-04-10 03:43:20.627454\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=93 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h32m0s\"\n  \"ts\"=\"2026-04-10 03:44:20.629732\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=94 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h33m0s\"\n  \"ts\"=\"2026-04-10 03:45:20.632735\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=95 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h34m0s\"\n  \"ts\"=\"2026-04-10 03:46:20.635585\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=96 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h35m0s\"\n  \"ts\"=\"2026-04-10 03:47:20.637668\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=97 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h36m0s\"\n  \"ts\"=\"2026-04-10 03:48:20.643845\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=98 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h37m0s\"\n  \"ts\"=\"2026-04-10 03:49:20.647679\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=99 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h38m0s\"\n  \"ts\"=\"2026-04-10 03:50:20.656164\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=100 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h39m0s\"\n  \"ts\"=\"2026-04-10 03:51:20.664785\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=101 \"elapsed\"=\"1h40m1s\"\n    STEP: creating a resource group @ 04/10/26 03:51:20.664\n  \"ts\"=\"2026-04-10 03:51:20.664925\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"pullsecret-test-2gn9mp\"\n    STEP: creating cluster parameters @ 04/10/26 03:51:24.256\n    STEP: creating customer resources @ 04/10/26 03:51:24.256\n    STEP: Creating the cluster @ 04/10/26 03:52:35.242\n  \"ts\"=\"2026-04-10 03:52:35.242550\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"pullsecret-hcp-cluster\" \"resourceGroup\"=\"pullsecret-test-2gn9mp\"\n    STEP: Creating the node pool @ 04/10/26 04:10:42.268\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\"=\"1m0s\" \"elapsed\"=\"1h23m0s\"\n  \"ts\"=\"2026-04-10 03:35:20.604098\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=85 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h24m0s\"\n  \"ts\"=\"2026-04-10 03:36:20.607920\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=86 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h25m0s\"\n  \"ts\"=\"2026-04-10 03:37:20.609946\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=87 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h26m0s\"\n  \"ts\"=\"2026-04-10 03:38:20.612631\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=88 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h27m0s\"\n  \"ts\"=\"2026-04-10 03:39:20.614440\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=89 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h28m0s\"\n  \"ts\"=\"2026-04-10 03:40:20.618754\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=90 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h29m0s\"\n  \"ts\"=\"2026-04-10 03:41:20.621295\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=91 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h30m0s\"\n  \"ts\"=\"2026-04-10 03:42:20.624572\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=92 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h31m0s\"\n  \"ts\"=\"2026-04-10 03:43:20.627454\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=93 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h32m0s\"\n  \"ts\"=\"2026-04-10 03:44:20.629732\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=94 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h33m0s\"\n  \"ts\"=\"2026-04-10 03:45:20.632735\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=95 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h34m0s\"\n  \"ts\"=\"2026-04-10 03:46:20.635585\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=96 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h35m0s\"\n  \"ts\"=\"2026-04-10 03:47:20.637668\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=97 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h36m0s\"\n  \"ts\"=\"2026-04-10 03:48:20.643845\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=98 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h37m0s\"\n  \"ts\"=\"2026-04-10 03:49:20.647679\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=99 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h38m0s\"\n  \"ts\"=\"2026-04-10 03:50:20.656164\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 1 identity containers but only 0 are assigned\" \"attempt\"=100 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h39m0s\"\n  \"ts\"=\"2026-04-10 03:51:20.664785\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=101 \"elapsed\"=\"1h40m1s\"\n    STEP: creating a resource group @ 04/10/26 03:51:20.664\n  \"ts\"=\"2026-04-10 03:51:20.664925\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"pullsecret-test-2gn9mp\"\n    STEP: creating cluster parameters @ 04/10/26 03:51:24.256\n    STEP: creating customer resources @ 04/10/26 03:51:24.256\n    STEP: Creating the cluster @ 04/10/26 03:52:35.242\n  \"ts\"=\"2026-04-10 03:52:35.242550\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"pullsecret-hcp-cluster\" \"resourceGroup\"=\"pullsecret-test-2gn9mp\"\n    STEP: Creating the node pool @ 04/10/26 04:10:42.268\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"a24b60c7c9a2e832f9ba51237dfb71341c05441a8987f985a1a432e7560a0348","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T04:11:17Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"a67bae610d136fbd27bf17b8e973ed3cb5e714c16c064a150a3c94cd5de9735e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"a1ffbe285ac5025d1bccac53237d64f36796874c70d5c5ae5d58b795494658ff","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001154720\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-f86lp7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-f86lp7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-f86lp7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001154720\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-f86lp7, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-f86lp7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-f86lp7, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"a7920103839d74099798f3a72a70190316cfc7bd355fdabcebdba1404165d89d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"8a441c3cc9a1724fd565ecc68f397cdfdb078c056f41b0ab5661bd4e719eb328","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"a7e7627d2db93b18f4a9025647bbf6e69f64db142cbf6cd6c372e22bbb37487d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"ccf1c285155e7641c812ec1a64a480027fe0f5935acb52b42c0d556cedc15270","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775790918 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:39\n  \"ts\"=\"2026-04-10 03:15:18.143083\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-create-a-cluster-with-default-autoscaling-and-a-nodepool-with-autoscaling-enabled-up-to-replica-limits|pid:823\"\n  \"ts\"=\"2026-04-10 03:15:18.149238\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: checking if the region supports availability zones @ 04/10/26 03:15:18.149\n    STEP: creating a resource group @ 04/10/26 03:15:19.661\n  \"ts\"=\"2026-04-10 03:15:19.661117\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\n    STEP: creating cluster parameters @ 04/10/26 03:15:23.102\n    STEP: creating customer resources @ 04/10/26 03:15:23.102\n    STEP: creating the HCP cluster @ 04/10/26 03:16:46.624\n  \"ts\"=\"2026-04-10 03:16:46.624184\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\n    STEP: verifying the cluster has default autoscaling parameters @ 04/10/26 03:27:40.525\n    STEP: getting admin credentials for the cluster @ 04/10/26 03:27:40.857\n    STEP: creating the AZ nodepool with 500 max replicas @ 04/10/26 03:36:33.03\n    STEP: verifying the AZ nodepool has the correct autoscaling configuration @ 04/10/26 03:46:22.114\n    STEP: verifying node count after AZ nodepool creation @ 04/10/26 03:46:30.42\n    STEP: updating the AZ nodepool max replicas from 500 to 4 before creating the next nodepool @ 04/10/26 03:46:31.463\n    STEP: verifying the AZ nodepool max replicas was updated to 4 @ 04/10/26 03:52:03.612\n    STEP: creating the no-AZ nodepool with 200 max replicas @ 04/10/26 03:52:03.976\n    STEP: verifying the no-AZ nodepool has the correct autoscaling configuration @ 04/10/26 04:01:52.182\n    STEP: verifying node count after no-AZ nodepool creation @ 04/10/26 04:01:53.049\n  \"ts\"=\"2026-04-10 04:01:54.362264\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 04:01:54.362332\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pgwxc7\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\n  \"ts\"=\"2026-04-10 04:01:54.650082\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-pgwxc7\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 04:01:56.684085\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4c275018436d67b080599c8133799f947e5f8f174147f1d7309198de43b9be47.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:01:56.684188\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4c275018436d67b080599c8133799f947e5f8f174147f1d7309198de43b9be47.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:01:56.684258\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 04:01:56.685776\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-114\"\n  \"ts\"=\"2026-04-10 04:01:56.685793\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\n  \"ts\"=\"2026-04-10 04:01:56.965161\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n  \"ts\"=\"2026-04-10 04:01:57.199959\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-114\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 04:01:57.230486\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pgwxc7\"\n  \"ts\"=\"2026-04-10 04:01:57.585359\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-114\" \"deployment\"=\"msi-deployment-114\"\n  \"ts\"=\"2026-04-10 04:01:57.751304\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 04:01:57.751435\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 04:01:57.751569\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775790918 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:39\n  \"ts\"=\"2026-04-10 03:15:18.143083\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-create-a-cluster-with-default-autoscaling-and-a-nodepool-with-autoscaling-enabled-up-to-replica-limits|pid:823\"\n  \"ts\"=\"2026-04-10 03:15:18.149238\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: checking if the region supports availability zones @ 04/10/26 03:15:18.149\n    STEP: creating a resource group @ 04/10/26 03:15:19.661\n  \"ts\"=\"2026-04-10 03:15:19.661117\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\n    STEP: creating cluster parameters @ 04/10/26 03:15:23.102\n    STEP: creating customer resources @ 04/10/26 03:15:23.102\n    STEP: creating the HCP cluster @ 04/10/26 03:16:46.624\n  \"ts\"=\"2026-04-10 03:16:46.624184\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\n    STEP: verifying the cluster has default autoscaling parameters @ 04/10/26 03:27:40.525\n    STEP: getting admin credentials for the cluster @ 04/10/26 03:27:40.857\n    STEP: creating the AZ nodepool with 500 max replicas @ 04/10/26 03:36:33.03\n    STEP: verifying the AZ nodepool has the correct autoscaling configuration @ 04/10/26 03:46:22.114\n    STEP: verifying node count after AZ nodepool creation @ 04/10/26 03:46:30.42\n    STEP: updating the AZ nodepool max replicas from 500 to 4 before creating the next nodepool @ 04/10/26 03:46:31.463\n    STEP: verifying the AZ nodepool max replicas was updated to 4 @ 04/10/26 03:52:03.612\n    STEP: creating the no-AZ nodepool with 200 max replicas @ 04/10/26 03:52:03.976\n    STEP: verifying the no-AZ nodepool has the correct autoscaling configuration @ 04/10/26 04:01:52.182\n    STEP: verifying node count after no-AZ nodepool creation @ 04/10/26 04:01:53.049\n  \"ts\"=\"2026-04-10 04:01:54.362264\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 04:01:54.362332\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pgwxc7\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\n  \"ts\"=\"2026-04-10 04:01:54.650082\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-pgwxc7\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 04:01:56.684085\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4c275018436d67b080599c8133799f947e5f8f174147f1d7309198de43b9be47.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:01:56.684188\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4c275018436d67b080599c8133799f947e5f8f174147f1d7309198de43b9be47.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:01:56.684258\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 04:01:56.685776\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-114\"\n  \"ts\"=\"2026-04-10 04:01:56.685793\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\n  \"ts\"=\"2026-04-10 04:01:56.965161\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n  \"ts\"=\"2026-04-10 04:01:57.199959\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-114\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 04:01:57.230486\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pgwxc7\"\n  \"ts\"=\"2026-04-10 04:01:57.585359\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-114\" \"deployment\"=\"msi-deployment-114\"\n  \"ts\"=\"2026-04-10 04:01:57.751304\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 04:01:57.751435\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 04:01:57.751569\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-6x7zd5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"b09095adfc5388cf6465bf217f411d8058591412682473f4eed642d06e23e862","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"ee3c35e66876b8446d35f42a776892654f3b6ea66fb1b4ab7ae0775c93d7a8c5","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001134810\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-78qtcm, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-78qtcm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-78qtcm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001134810\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-78qtcm, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-78qtcm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-78qtcm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded"}},{"row_id":"b55a1866d46e1845c28ec26fdd95928cd4a37c970977630d5e0a194f67aa2de7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"74a2f372f73f571cf44d910b50c9e1559c9dfb4de2568689b9ffb24d2fdb9485","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775787080 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group [PreLaunchSetup:None, ARO-HCP-RP-API-Compatible, Importance:Critical, Positivity:Positive]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35\n  \"ts\"=\"2026-04-10 02:11:20.147352\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=3 \"specID\"=\"Customer-should-be-able-to-create-several-HCP-clusters-in-their-customer-resource-group,-but-not-in-the-same-managed-resource-group|pid:123\"\n  \"ts\"=\"2026-04-10 02:11:20.304024\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=3 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a shared customer resource group @ 04/10/26 02:11:20.304\n  \"ts\"=\"2026-04-10 02:11:20.304178\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"customer-rg-rpmwqz\"\n    STEP: creating customer infrastructure and managed identities for first cluster @ 04/10/26 02:11:25.616\n    STEP: creating customer infrastructure and managed identities for second cluster @ 04/10/26 02:12:46.416\n    STEP: starting creation of both clusters in parallel @ 04/10/26 02:14:09.067\n  \"ts\"=\"2026-04-10 02:14:09.067714\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster\" \"resourceGroup\"=\"customer-rg-rpmwqz\"\n  \"ts\"=\"2026-04-10 02:14:19.368932\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster2\" \"resourceGroup\"=\"customer-rg-rpmwqz\"\n    STEP: waiting for first cluster to complete creation @ 04/10/26 02:14:28.135\n    STEP: waiting for second cluster to complete creation @ 04/10/26 02:45:48.132\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775787080 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group [PreLaunchSetup:None, ARO-HCP-RP-API-Compatible, Importance:Critical, Positivity:Positive]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35\n  \"ts\"=\"2026-04-10 02:11:20.147352\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=3 \"specID\"=\"Customer-should-be-able-to-create-several-HCP-clusters-in-their-customer-resource-group,-but-not-in-the-same-managed-resource-group|pid:123\"\n  \"ts\"=\"2026-04-10 02:11:20.304024\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=3 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a shared customer resource group @ 04/10/26 02:11:20.304\n  \"ts\"=\"2026-04-10 02:11:20.304178\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"customer-rg-rpmwqz\"\n    STEP: creating customer infrastructure and managed identities for first cluster @ 04/10/26 02:11:25.616\n    STEP: creating customer infrastructure and managed identities for second cluster @ 04/10/26 02:12:46.416\n    STEP: starting creation of both clusters in parallel @ 04/10/26 02:14:09.067\n  \"ts\"=\"2026-04-10 02:14:09.067714\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster\" \"resourceGroup\"=\"customer-rg-rpmwqz\"\n  \"ts\"=\"2026-04-10 02:14:19.368932\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-hcp-cluster2\" \"resourceGroup\"=\"customer-rg-rpmwqz\"\n    STEP: waiting for first cluster to complete creation @ 04/10/26 02:14:28.135\n    STEP: waiting for second cluster to complete creation @ 04/10/26 02:45:48.132\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"c46e88dd29188780dae1691e1926777dd9f63d4a74ee96f16e7c00ad41bdb5b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"ae1f25418990f1a3fcdf9ce89f3756ee2927b16ebcbdfc62e13edc53b8bfa567","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775792709 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n  \"ts\"=\"2026-04-10 03:45:09.057993\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-not-be-able-to-perform-various-invalid-operations-on-cluster-resources|pid:930\"\n  \"ts\"=\"2026-04-10 03:45:09.065440\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 03:45:09.065\n  \"ts\"=\"2026-04-10 03:45:09.065489\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-szl4vm\"\n    STEP: creating cluster parameters @ 04/10/26 03:45:13.668\n    STEP: creating customer resources @ 04/10/26 03:45:13.668\n    STEP: creating the HCP cluster @ 04/10/26 03:46:32.455\n  \"ts\"=\"2026-04-10 03:46:32.456038\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-szl4vm\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775792709 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n  \"ts\"=\"2026-04-10 03:45:09.057993\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-not-be-able-to-perform-various-invalid-operations-on-cluster-resources|pid:930\"\n  \"ts\"=\"2026-04-10 03:45:09.065440\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 03:45:09.065\n  \"ts\"=\"2026-04-10 03:45:09.065489\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-szl4vm\"\n    STEP: creating cluster parameters @ 04/10/26 03:45:13.668\n    STEP: creating customer resources @ 04/10/26 03:45:13.668\n    STEP: creating the HCP cluster @ 04/10/26 03:46:32.455\n  \"ts\"=\"2026-04-10 03:46:32.456038\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-szl4vm\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"d0f3453b7cefec26c94c4f23944d1840f75cf628b642d4f9d582a550ec44c923","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"7a3430cc1684e21012eb2689e5bed9d9e16bc291639be081252556bdf6682079","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775791434 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n  \"ts\"=\"2026-04-10 03:23:54.336107\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:917\"\n  \"ts\"=\"2026-04-10 03:23:54.342933\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 03:23:54.342\n  \"ts\"=\"2026-04-10 03:23:54.343003\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\n    STEP: creating cluster parameters @ 04/10/26 03:23:59.225\n    STEP: creating customer resources @ 04/10/26 03:23:59.226\n    STEP: creating the HCP cluster @ 04/10/26 03:25:13.715\n  \"ts\"=\"2026-04-10 03:25:13.715863\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\n    STEP: creating a storage account for shoebox logs @ 04/10/26 03:36:39.988\n  \"ts\"=\"2026-04-10 03:37:03.480045\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoebox5f8x7q\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-6gpfxz/providers/Microsoft.Storage/storageAccounts/shoebox5f8x7q\"\n    STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 03:37:03.48\n  \"ts\"=\"2026-04-10 03:37:18.683593\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n    STEP: waiting for log containers to appear in the storage account @ 04/10/26 03:37:18.683\n  \"ts\"=\"2026-04-10 04:04:19.006457\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoebox5f8x7q\" \"timestamp\"=\"2026-04-10T04:04:19Z\"\n  \"ts\"=\"2026-04-10 04:04:19.006655\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 04:04:19.006753\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-72tcfz\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\n  \"ts\"=\"2026-04-10 04:04:19.290758\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-72tcfz\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 04:04:21.555661\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:04:21.555765\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:04:21.555825\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 04:04:21.557374\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-17\"\n  \"ts\"=\"2026-04-10 04:04:21.557391\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\n  \"ts\"=\"2026-04-10 04:04:21.712755\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-17\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 04:04:21.844317\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-10 04:04:22.121298\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-72tcfz\"\n  \"ts\"=\"2026-04-10 04:04:22.264844\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-17\" \"deployment\"=\"msi-deployment-17\"\n  \"ts\"=\"2026-04-10 04:04:22.422889\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 04:04:22.423005\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 04:04:22.423122\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775791434 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n  \"ts\"=\"2026-04-10 03:23:54.336107\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:917\"\n  \"ts\"=\"2026-04-10 03:23:54.342933\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 04/10/26 03:23:54.342\n  \"ts\"=\"2026-04-10 03:23:54.343003\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\n    STEP: creating cluster parameters @ 04/10/26 03:23:59.225\n    STEP: creating customer resources @ 04/10/26 03:23:59.226\n    STEP: creating the HCP cluster @ 04/10/26 03:25:13.715\n  \"ts\"=\"2026-04-10 03:25:13.715863\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\n    STEP: creating a storage account for shoebox logs @ 04/10/26 03:36:39.988\n  \"ts\"=\"2026-04-10 03:37:03.480045\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoebox5f8x7q\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-6gpfxz/providers/Microsoft.Storage/storageAccounts/shoebox5f8x7q\"\n    STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 03:37:03.48\n  \"ts\"=\"2026-04-10 03:37:18.683593\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n    STEP: waiting for log containers to appear in the storage account @ 04/10/26 03:37:18.683\n  \"ts\"=\"2026-04-10 04:04:19.006457\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoebox5f8x7q\" \"timestamp\"=\"2026-04-10T04:04:19Z\"\n  \"ts\"=\"2026-04-10 04:04:19.006655\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-10 04:04:19.006753\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-72tcfz\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\n  \"ts\"=\"2026-04-10 04:04:19.290758\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-72tcfz\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-10 04:04:21.555661\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:04:21.555765\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n  \"ts\"=\"2026-04-10 04:04:21.555825\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-10 04:04:21.557374\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-17\"\n  \"ts\"=\"2026-04-10 04:04:21.557391\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\n  \"ts\"=\"2026-04-10 04:04:21.712755\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-17\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-10 04:04:21.844317\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n  \"ts\"=\"2026-04-10 04:04:22.121298\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-72tcfz\"\n  \"ts\"=\"2026-04-10 04:04:22.264844\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-17\" \"deployment\"=\"msi-deployment-17\"\n  \"ts\"=\"2026-04-10 04:04:22.422889\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-10 04:04:22.423005\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-10 04:04:22.423122\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-6gpfxz\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"d526c22777be34fadd3dc4dae895d53cea7ce885397639ff621cae9fad230614","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"b7bb55f3055dea91e4e2e0a35768691dd8a3bb6770c7125349f05399ef297db8","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"ontainers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n  \"ts\"=\"2026-04-10 03:53:20.586689\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n  \"ts\"=\"2026-04-10 03:54:20.589807\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n  \"ts\"=\"2026-04-10 03:55:20.592550\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n  \"ts\"=\"2026-04-10 03:56:20.596065\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n  \"ts\"=\"2026-04-10 03:57:20.598781\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n  \"ts\"=\"2026-04-10 03:58:20.601284\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n  \"ts\"=\"2026-04-10 03:59:20.604055\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n  \"ts\"=\"2026-04-10 04:00:20.606820\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n  \"ts\"=\"2026-04-10 04:01:20.609895\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n  \"ts\"=\"2026-04-10 04:02:20.612685\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n  \"ts\"=\"2026-04-10 04:03:20.615863\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n  \"ts\"=\"2026-04-10 04:04:20.618713\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n  \"ts\"=\"2026-04-10 04:05:20.622636\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n  \"ts\"=\"2026-04-10 04:06:20.624843\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n  \"ts\"=\"2026-04-10 04:07:20.628107\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n  \"ts\"=\"2026-04-10 04:08:20.630458\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=118 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h57m0s\"\n  \"ts\"=\"2026-04-10 04:09:20.639551\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=2 \"attempt\"=119 \"elapsed\"=\"1h58m0s\"\n    STEP: creating a resource group @ 04/10/26 04:09:20.639\n  \"ts\"=\"2026-04-10 04:09:20.639659\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-l4gfm9\"\n    STEP: creating customer resources @ 04/10/26 04:09:24.764\n    STEP: seeding HCP cluster @ 04/10/26 04:10:34.935\n  \"ts\"=\"2026-04-10 04:10:34.935456\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-cluster\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-l4gfm9\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nontainers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=102 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h41m0s\"\n  \"ts\"=\"2026-04-10 03:53:20.586689\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=103 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h42m0s\"\n  \"ts\"=\"2026-04-10 03:54:20.589807\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=104 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h43m0s\"\n  \"ts\"=\"2026-04-10 03:55:20.592550\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=105 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h44m0s\"\n  \"ts\"=\"2026-04-10 03:56:20.596065\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=106 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h45m0s\"\n  \"ts\"=\"2026-04-10 03:57:20.598781\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=107 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h46m0s\"\n  \"ts\"=\"2026-04-10 03:58:20.601284\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=108 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h47m0s\"\n  \"ts\"=\"2026-04-10 03:59:20.604055\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 0 are assigned\" \"attempt\"=109 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h48m0s\"\n  \"ts\"=\"2026-04-10 04:00:20.606820\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=110 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h49m0s\"\n  \"ts\"=\"2026-04-10 04:01:20.609895\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=111 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h50m0s\"\n  \"ts\"=\"2026-04-10 04:02:20.612685\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=112 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h51m0s\"\n  \"ts\"=\"2026-04-10 04:03:20.615863\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=113 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h52m0s\"\n  \"ts\"=\"2026-04-10 04:04:20.618713\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=114 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h53m0s\"\n  \"ts\"=\"2026-04-10 04:05:20.622636\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=115 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h54m0s\"\n  \"ts\"=\"2026-04-10 04:06:20.624843\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=116 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h55m0s\"\n  \"ts\"=\"2026-04-10 04:07:20.628107\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=117 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h56m0s\"\n  \"ts\"=\"2026-04-10 04:08:20.630458\" \"level\"=0 \"msg\"=\"Not enough free identity containers, waiting to retry\" \"error\"=\"not enough free identity containers: requested 2 identity containers but only 1 are assigned\" \"attempt\"=118 \"retryIn\"=\"1m0s\" \"elapsed\"=\"1h57m0s\"\n  \"ts\"=\"2026-04-10 04:09:20.639551\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=2 \"attempt\"=119 \"elapsed\"=\"1h58m0s\"\n    STEP: creating a resource group @ 04/10/26 04:09:20.639\n  \"ts\"=\"2026-04-10 04:09:20.639659\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-l4gfm9\"\n    STEP: creating customer resources @ 04/10/26 04:09:24.764\n    STEP: seeding HCP cluster @ 04/10/26 04:10:34.935\n  \"ts\"=\"2026-04-10 04:10:34.935456\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"basic-cluster\" \"resourceGroup\"=\"rg-cluster-nsg-subnet-reuse-l4gfm9\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e25ec9b59861203758495f83c2e56cc13b717c4f54ebdfd01273212e11736882","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","failed_at":"e2e","signature_id":"de4a5ce3ac3c5e9aea9290d16182f031cb6e3c37df7abdcce982c39a9f3fc408","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00096a480\u003e: \n    failed waiting for hcpCluster=\"cluster-stable-4-20-hcbxqh\" in resourcegroup=\"rg-stable-4-20-tpn8xl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-hcbxqh in resource group rg-stable-4-20-tpn8xl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:129]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00096a480\u003e: \n    failed waiting for hcpCluster=\"cluster-stable-4-20-hcbxqh\" in resourcegroup=\"rg-stable-4-20-tpn8xl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-hcbxqh in resource group rg-stable-4-20-tpn8xl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":21,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":21,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042422454114062336","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-10T02:00:34Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}